Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/envvars.rst
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,23 @@ Context Parallelism
FP8 Configuration
^^^^^^^^^^^^^^^^^

.. envvar:: NVTE_FP8_BLOCK_QUANTIZATION_GROUPED

:Type: ``int`` (0 or 1)
:Default: ``1``
:Description: Enable grouped 1D blockwise FP8 quantization on Hopper for compatible
lists of at least two BF16 matrices. Inputs must share K, but M may differ, and
all quantizers must request both rowwise and columnwise outputs
with matching FP8 dtype, epsilon, and power-of-two scaling options.
The existing split and multi-tensor APIs select this path
automatically. Other configurations retain their existing path.
Set to ``0`` to disable the optimization for comparison.

M may vary across tensors and between API calls. CUDA Graph replay
uses the shapes and buffer addresses recorded during capture;
input values may change in place between replays. Changing M
between replays requires updating or recapturing the graph.

.. envvar:: NVTE_UNFUSED_FP8_UPDATE

:Type: ``int`` (0 or 1)
Expand Down
257 changes: 257 additions & 0 deletions tests/pytorch/test_float8blockwisetensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -641,3 +641,260 @@ def test_clone_detach(
# Make sure we are not trivially passing tests
with pytest.raises(AssertionError):
torch.testing.assert_close(x_clone.dequantize(), -x_hp, **_tols[fp8_dtype])


def _snapshot_blockwise_outputs(outputs, shapes, block_dim):
snapshots = []
for out, (m, k) in zip(outputs, shapes):
for direction in ("rowwise", "columnwise"):
data = getattr(out, f"_{direction}_data")
scale = getattr(out, f"_{direction}_scale_inv")
if data is None:
snapshots.append(None)
continue
# Consume on the caller stream immediately, before synchronizing.
snapshots.append(data.view(torch.uint8).clone())
if block_dim == 2:
rows, cols = (m + 127) // 128, (k + 127) // 128
if direction == "columnwise":
rows, cols = cols, rows
elif direction == "rowwise":
rows, cols = (k + 127) // 128, m
else:
rows, cols = (m + 127) // 128, k
# Scale storage includes unspecified GEMM-alignment padding.
snapshots.append(scale[:rows, :cols].clone())
return snapshots


def _assert_blockwise_snapshots_equal(actual, expected):
assert len(actual) == len(expected)
for a, b in zip(actual, expected):
if a is None or b is None:
assert a is None and b is None
else:
torch.testing.assert_close(a, b, rtol=0, atol=0)


@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe)
@pytest.mark.parametrize(
"splits", [[128, 0, 256, 384] * 4, [0, 1, 127, 129, 255], [129] * 33, [0, 0], [129]]
)
@pytest.mark.parametrize("k", [256, 257])
@pytest.mark.parametrize("bulk", [True, False])
@pytest.mark.parametrize("fp8_dtype", [DType.kFloat8E4M3, DType.kFloat8E5M2])
@pytest.mark.parametrize("pow2_scale", [False, True])
def test_split_quantize_grouped(monkeypatch, splits, k, bulk, fp8_dtype, pow2_scale):
"""Variable M, empty/partial tiles, chunk boundaries, and caller ordering."""
if get_device_compute_capability()[0] != 9:
pytest.skip("Grouped BF16 blockwise path targets Hopper")
shapes = [(m, k) for m in splits]
quantizers = [
Float8BlockQuantizer(
fp8_dtype=fp8_dtype,
rowwise=True,
columnwise=True,
block_scaling_dim=1,
force_pow_2_scales=pow2_scale,
amax_epsilon=0.01,
)
for _ in splits
]
caller = torch.cuda.Stream()
with torch.cuda.stream(caller):
for _ in range(2):
x = torch.randn(sum(splits), k, dtype=torch.bfloat16, device="cuda")
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0")
reference = tex.split_quantize(x, splits, quantizers, disable_bulk_allocation=not bulk)
expected = _snapshot_blockwise_outputs(reference, shapes, 1)
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1")
outputs = tex.split_quantize(x, splits, quantizers, disable_bulk_allocation=not bulk)
actual = _snapshot_blockwise_outputs(outputs, shapes, 1)
assert torch.cuda.current_stream() == caller
caller.synchronize()
_assert_blockwise_snapshots_equal(actual, expected)


@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe)
@pytest.mark.parametrize("shape", [(128, 384), (129, 385)])
@pytest.mark.parametrize("fp8_dtype", [DType.kFloat8E4M3, DType.kFloat8E5M2])
@pytest.mark.parametrize("pow2_scale", [False, True])
@pytest.mark.parametrize("case", ["nan", "inf", "zero", "tiny", "scaling_boundary"])
def test_split_quantize_grouped_special_values(monkeypatch, shape, fp8_dtype, pow2_scale, case):
"""Match scalar amax for special values in both aligned and partial tiles."""
if get_device_compute_capability()[0] != 9:
pytest.skip("Grouped BF16 blockwise path targets Hopper")
m, k = shape
if case == "nan":
values = [float("nan")]
elif case == "inf":
values = [float("inf"), -float("inf"), 1.0, -1.0]
elif case == "zero":
values = [0.0, -0.0]
elif case == "tiny":
# Smallest/largest BF16 subnormals and smallest normal, with both signs.
values = [sign * n * 2.0**-133 for n in (1, 127, 128) for sign in (1.0, -1.0)]
else:
fp8_max = 448.0 if fp8_dtype == DType.kFloat8E4M3 else 57344.0
boundary = fp8_max / 128.0
ulp = math.ldexp(1.0, math.frexp(boundary)[1] - 8)
values = [sign * (boundary + delta * ulp) for sign in (1.0, -1.0) for delta in (-1, 0, 1)]
x = torch.empty(2 * m, k, dtype=torch.bfloat16, device="cuda")
tile = 0
for expert in x.split(m):
for row in range(0, m, 128):
for col in range(0, k, 128):
# Uniform tiles keep each candidate visible as a block amax.
expert[row : row + 128, col : col + 128].fill_(values[tile % len(values)])
tile += 1
if case == "nan":
# Mix finite values with NaNs, retaining all-NaN rows and columns.
x[1::2, 1::2] = 1.0
quantizers = [
Float8BlockQuantizer(
fp8_dtype=fp8_dtype,
rowwise=True,
columnwise=True,
block_scaling_dim=1,
force_pow_2_scales=pow2_scale,
amax_epsilon=0.0,
)
for _ in range(2)
]
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0")
reference = tex.split_quantize(x, [m, m], quantizers)
expected = _snapshot_blockwise_outputs(reference, [shape, shape], 1)
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1")
outputs = tex.split_quantize(x, [m, m], quantizers)
actual = _snapshot_blockwise_outputs(outputs, [shape, shape], 1)
torch.cuda.synchronize()
_assert_blockwise_snapshots_equal(actual, expected)


@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe)
@pytest.mark.parametrize(
"usage,block_dim,dtype,mixed_options",
[
((True, False), 1, torch.bfloat16, False),
((False, True), 1, torch.bfloat16, False),
((True, True), 2, torch.bfloat16, False),
((True, True), 1, torch.float16, False),
((True, True), 1, torch.bfloat16, True),
],
)
def test_split_quantize_grouped_fallback(monkeypatch, usage, block_dim, dtype, mixed_options):
"""Keep single-direction, 2D scaling, other dtypes and per-expert options."""
splits, k = [128, 256, 0], 256
quantizers = [
Float8BlockQuantizer(
fp8_dtype=DType.kFloat8E4M3,
rowwise=usage[0],
columnwise=usage[1],
block_scaling_dim=block_dim,
amax_epsilon=(0.1 * i if mixed_options else 0.0),
)
for i in range(len(splits))
]
x = torch.randn(sum(splits), k, dtype=dtype, device="cuda")
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0")
reference = tex.split_quantize(x, splits, quantizers)
expected = _snapshot_blockwise_outputs(reference, [(m, k) for m in splits], block_dim)
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1")
outputs = tex.split_quantize(x, splits, quantizers)
actual = _snapshot_blockwise_outputs(outputs, [(m, k) for m in splits], block_dim)
torch.cuda.synchronize()
_assert_blockwise_snapshots_equal(actual, expected)


@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe)
@pytest.mark.parametrize("common_k", [True, False])
def test_multi_tensor_quantize_grouped(monkeypatch, common_k):
"""Support separate allocations; mixed K must retain the existing path."""
shapes = [(129, 256), (256, 256 if common_k else 384)]
inputs = [torch.randn(shape, dtype=torch.bfloat16, device="cuda") for shape in shapes]
quantizers = [
Float8BlockQuantizer(
fp8_dtype=DType.kFloat8E4M3,
rowwise=True,
columnwise=True,
block_scaling_dim=1,
)
for _ in inputs
]
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0")
reference = tex.multi_tensor_quantize(inputs, quantizers)
expected = _snapshot_blockwise_outputs(reference, shapes, 1)
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1")
outputs = tex.multi_tensor_quantize(inputs, quantizers)
actual = _snapshot_blockwise_outputs(outputs, shapes, 1)
torch.cuda.synchronize()
_assert_blockwise_snapshots_equal(actual, expected)


@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe)
def test_split_quantize_grouped_dispatch(monkeypatch):
"""Ensure a qualifying call actually executes one grouped CUDA kernel."""
if get_device_compute_capability()[0] != 9:
pytest.skip("Grouped BF16 blockwise path targets Hopper")
splits, k = [128, 0, 256, 384], 256
quantizers = [
Float8BlockQuantizer(
fp8_dtype=DType.kFloat8E4M3,
rowwise=True,
columnwise=True,
block_scaling_dim=1,
)
for _ in splits
]
x = torch.randn(sum(splits), k, dtype=torch.bfloat16, device="cuda")
monkeypatch.delenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", raising=False)
tex.split_quantize(x, splits, quantizers)
torch.cuda.synchronize()
with torch.profiler.profile(
activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA]
) as profile:
tex.split_quantize(x, splits, quantizers)
torch.cuda.synchronize()
kernels = [
event
for event in profile.events()
if "grouped_block_scaled_1d_cast_transpose_kernel" in event.name
]
assert len(kernels) == 1, "Expected one grouped kernel; rebuild the TE core library as well"


@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe)
def test_split_quantize_grouped_cuda_graph(monkeypatch):
"""Replay with fixed shapes and buffer addresses, changing only input values."""
if get_device_compute_capability()[0] != 9:
pytest.skip("Grouped BF16 blockwise path targets Hopper")
splits, k = [0, 129, 256, 511], 256
quantizers = [
Float8BlockQuantizer(
fp8_dtype=DType.kFloat8E4M3,
rowwise=True,
columnwise=True,
block_scaling_dim=1,
)
for _ in splits
]
x = torch.ones(sum(splits), k, dtype=torch.bfloat16, device="cuda")
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1")
warmup = torch.cuda.Stream()
warmup.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup):
for _ in range(3):
tex.split_quantize(x, splits, quantizers)
torch.cuda.current_stream().wait_stream(warmup)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
outputs = tex.split_quantize(x, splits, quantizers)
for value in (0.0, 0.5, 2.0, -3.0):
x.fill_(value)
graph.replay()
actual = _snapshot_blockwise_outputs(outputs, [(m, k) for m in splits], 1)
monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0")
reference = tex.split_quantize(x, splits, quantizers)
expected = _snapshot_blockwise_outputs(reference, [(m, k) for m in splits], 1)
torch.cuda.synchronize()
_assert_blockwise_snapshots_equal(actual, expected)
9 changes: 9 additions & 0 deletions transformer_engine/common/cast/cast.cu
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ void nvte_multi_tensor_quantize(const NVTETensor *inputs, NVTETensor *outputs,
NVTE_API_CALL(nvte_multi_tensor_quantize);
using namespace transformer_engine;

QuantizationConfig config;
if (quant_configs != nullptr) {
config = *reinterpret_cast<QuantizationConfig *>(quant_configs);
}
if (detail::try_grouped_quantize_transpose_vector_blockwise(inputs, outputs, num_tensors, config,
stream)) {
return;
}

constexpr bool IS_ACT = false;

const size_t num_streams = nvte_get_num_compute_streams();
Expand Down
7 changes: 7 additions & 0 deletions transformer_engine/common/include/transformer_engine/cast.h
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,13 @@ void nvte_group_requantize(const NVTEGroupedTensor input, NVTEGroupedTensor outp
cudaStream_t stream);

/*! \brief Casts multiple input tensors to quantized output tensors.
*
* On Hopper, compatible BF16 matrices with common K and both GEMM-ready 1D
* block-scaled FP8 outputs use grouped quantization. M may vary per tensor.
* Other configurations retain their existing quantization path.
* With CUDA Graph capture, shapes and buffer addresses are fixed for replay;
* input values may change in place. Different M values between API calls do not
* imply support for changing M between replays of the same captured graph.
*
* \param[in] inputs List of input tensors to be cast.
* \param[in,out] outputs List of output quantized tensors.
Expand Down
8 changes: 8 additions & 0 deletions transformer_engine/common/transpose/cast_transpose.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ enum class FP8BlockwiseColumnwiseOption {
COLUMNWISE_COMPACT
};

// Try grouped BF16-to-FP8 quantization with both GEMM-ready orientations.
// All inputs must have the same K; M may vary. Returns false before launching
// work if any tensor/configuration needs the existing per-tensor path.
bool try_grouped_quantize_transpose_vector_blockwise(const NVTETensor *inputs, NVTETensor *outputs,
size_t num_tensors,
const QuantizationConfig &config,
cudaStream_t stream);

void quantize_transpose_vector_blockwise(const SimpleTensor &input, SimpleTensor &scale_inv,
SimpleTensor &scale_inv_t, SimpleTensor &output,
SimpleTensor &output_t, const float epsilon,
Expand Down
Loading