diff --git a/3rdparty/nccl-extensions b/3rdparty/nccl-extensions index a9f207c2263..998d9223e00 160000 --- a/3rdparty/nccl-extensions +++ b/3rdparty/nccl-extensions @@ -1 +1 @@ -Subproject commit a9f207c2263b4af1622a1d902b3e608fe17be0eb +Subproject commit 998d9223e00c20ee515ceeccf4f0f086ee8038cd diff --git a/docs/envvars.rst b/docs/envvars.rst index 46b70bbe46a..62b8ac5f4f0 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -342,6 +342,12 @@ Kernel Configuration :Default: ``0`` :Description: Disable NVRTC (CUDA Runtime Compilation) support. When set to ``1``, runtime kernel compilation is disabled. Existing transpose operations select their static fallback automatically. NVFP4 4over6 quantization, fused softmax, and normalization paths require their corresponding ``NVTE_BUILD_LEGACY_STATIC_NVFP4``, ``NVTE_BUILD_LEGACY_STATIC_FUSED_SOFTMAX``, or ``NVTE_BUILD_LEGACY_STATIC_NORM`` CMake option to have been enabled when the library was built; otherwise no static fallback is available. +.. envvar:: NVTE_MXFP8_PER_TILE_SWIZZLE + + :Type: ``int`` (0 or 1) + :Default: ``0`` + :Description: Swizzle MXFP8 scaling factors for GEMM with kernels that give each 128-row tile of scaling factors (row-wise) or each 128-column by 32-row block (column-wise) its own thread block, instead of batching 32 tiles in one block. This lowers the latency of swizzling the scaling factors of tensors with few tiles, such as activations. + .. envvar:: NVTE_USE_CUTLASS_GROUPED_GEMM :Type: ``int`` (0 or 1) diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 2a486450a20..7ace265048c 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -70,7 +70,7 @@ NVTE_GDN2_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/py NVTE_GDP_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdp_attention.xml $TE_PATH/tests/pytorch/attention/test_gdp_attention.py || test_fail "test_gdp_attention.py" NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention_deterministic.xml $TE_PATH/tests/pytorch/attention/test_attention.py || test_fail "NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 test_attention.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_linear_mxfp8_attention.xml $TE_PATH/tests/pytorch/attention/test_linear_mxfp8_attention.py || test_fail "test_linear_mxfp8_attention.py" -python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_mla_q_uproj.xml $TE_PATH/tests/pytorch/attention/test_fused_mla_q_uproj.py || test_fail "test_fused_mla_q_uproj.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_mla.xml $TE_PATH/tests/pytorch/attention/test_fused_mla.py || test_fail "test_fused_mla.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_kv_cache.xml $TE_PATH/tests/pytorch/attention/test_kv_cache.py || test_fail "test_kv_cache.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cu_seqlens_cache.xml $TE_PATH/tests/pytorch/attention/test_cu_seqlens_cache.py || test_fail "test_cu_seqlens_cache.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mixed_thd_attention.xml $TE_PATH/tests/pytorch/attention/test_mixed_thd_attention.py || test_fail "test_mixed_thd_attention.py" diff --git a/qa/L3_pytorch_FA_versions_test/test.sh b/qa/L3_pytorch_FA_versions_test/test.sh index e7fd3189f35..266abff774d 100644 --- a/qa/L3_pytorch_FA_versions_test/test.sh +++ b/qa/L3_pytorch_FA_versions_test/test.sh @@ -36,8 +36,8 @@ export FLASH_ATTN_CUDA_ARCHS=$sm_arch CP_FA_VERSION="" if [ $sm_arch -gt 90 ] then - FA_versions=(4.0.0b11) - CP_FA_VERSION="4.0.0b11" + FA_versions=(4.0.0b33) + CP_FA_VERSION="4.0.0b33" elif [ $sm_arch -eq 90 ] then FA_versions=(3.0.0b1) @@ -66,13 +66,13 @@ do elif [[ "${fa_version}" == 4.* ]] then export NVTE_FLASH_ATTN_V4=1 - # FA4 is intentionally last in every version array. Its b11 test pin needs - # CUTLASS DSL 4.4.2, so replace the image-matched stack only for this final + # FA4 is intentionally last in every version array. CUTLASS DSL 4.8.0 supports + # SM107, so replace the image-matched stack only for this final # iteration; later iterations would otherwise need that stack restored. pip3 uninstall -y nvidia-cutlass-dsl nvidia-cutlass-dsl-libs-base \ nvidia-cutlass-dsl-libs-cu12 nvidia-cutlass-dsl-libs-cu13 \ || error_exit "Failed to isolate CUTLASS DSL for Flash Attention $fa_version" - pip3 install flash-attn-4==${fa_version} nvidia-cutlass-dsl[cu13]==4.4.2 \ + pip3 install flash-attn-4==${fa_version} nvidia-cutlass-dsl[cu13]==4.8.0 \ --no-build-isolation || error_exit "Failed to install Flash Attention $fa_version" else export NVTE_FLASH_ATTN_V3=1 @@ -135,6 +135,13 @@ do fi NVTE_TORCH_COMPILE=0 NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 python3 -m pytest -v -s --junitxml=$XML_ATTN $TE_PATH/tests/pytorch/attention/test_attention.py || test_fail "test_attention.py (FA $fa_version)" fi + + if [[ "${fa_version}" == 4.* ]]; then + XML_BACKEND="$XML_LOG_DIR/pytest_test_attention_backend_selection_fa${fa_tag}.xml" + NVTE_TORCH_COMPILE=0 python3 -m pytest -v -s --junitxml="$XML_BACKEND" \ + "$TE_PATH/tests/pytorch/attention/test_attention_backend_selection.py" \ + || test_fail "FA4 backend-selection tests (FA $fa_version)" + fi done if [ "$RET" -ne 0 ]; then diff --git a/tests/cpp/operator/test_cast_mxfp8.cu b/tests/cpp/operator/test_cast_mxfp8.cu index 9368d4030b5..16fb977b788 100644 --- a/tests/cpp/operator/test_cast_mxfp8.cu +++ b/tests/cpp/operator/test_cast_mxfp8.cu @@ -746,14 +746,13 @@ std::vector scaling_directions_2d_quantize = { MXFP82DScalingDirection::Bidirectional, }; -// Only GeLU activation tests are supported std::vector Activation_types = { ActivationType::Identity, ActivationType::GeLU, - // ActivationType::SiLU, - // ActivationType::ReLU, - // ActivationType::QGeLU, - // ActivationType::SReLU, + ActivationType::SiLU, + ActivationType::ReLU, + ActivationType::QGeLU, + ActivationType::SReLU, }; class FusedCastMXFP8TestSuite : public ::testing::TestWithParam diff --git a/tests/cpp/operator/test_cast_mxfp8_grouped.cu b/tests/cpp/operator/test_cast_mxfp8_grouped.cu index c2309f7aa4b..03ec1acd001 100644 --- a/tests/cpp/operator/test_cast_mxfp8_grouped.cu +++ b/tests/cpp/operator/test_cast_mxfp8_grouped.cu @@ -657,10 +657,10 @@ std::vector processing_methods = { std::vector activation_kinds = { ActivationKind::Identity, ActivationKind::GeLU, - // ActivationKind::SiLU, - // ActivationKind::ReLU, - // ActivationKind::QGeLU, - // ActivationKind::SReLU, + ActivationKind::SiLU, + ActivationKind::ReLU, + ActivationKind::QGeLU, + ActivationKind::SReLU, }; enum ScalingDirection { @@ -1012,7 +1012,7 @@ INSTANTIATE_TEST_SUITE_P( GroupedFusedCastMXFP8TestSuite, ::testing::Combine( ::testing::Values(ProcessingMethod::CAST_DBIAS_DACT), - ::testing::Values(ActivationKind::GeLU), + ::testing::ValuesIn(activation_kinds), ::testing::ValuesIn(scaling_directions), ::testing::ValuesIn(input_config_same_both_multichunk), ::testing::Values(DType::kBFloat16), diff --git a/tests/cpp/operator/test_multi_swizzle.cu b/tests/cpp/operator/test_multi_swizzle.cu index d8b8211e5b7..4bd1800cea2 100644 --- a/tests/cpp/operator/test_multi_swizzle.cu +++ b/tests/cpp/operator/test_multi_swizzle.cu @@ -300,14 +300,22 @@ void performTestMultiTensorRoundtrip(const int num_tensors, const size_t M, cons // Test suites // =================================================================== +// Sets NVTE_MXFP8_PER_TILE_SWIZZLE while it lives, so that MXFP8 swizzles use the per-tile +// kernels (enable) or the default ones. +class PerTileSwizzle { + public: + explicit PerTileSwizzle(bool enable) { + setenv("NVTE_MXFP8_PER_TILE_SWIZZLE", enable ? "1" : "0", 1); + } + ~PerTileSwizzle() { unsetenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } +}; + class MultiTensorSwizzleTestSuite - : public ::testing::TestWithParam> {}; + : public ::testing::TestWithParam, bool>> {}; TEST_P(MultiTensorSwizzleTestSuite, TestMultiTensorSwizzle) { - const auto num_tensors = std::get<0>(GetParam()); - const auto M = std::get<1>(GetParam()); - const auto K = std::get<2>(GetParam()); - const auto rowwise = std::get<3>(GetParam()); + const auto& [num_tensors, M, K, rowwise] = std::get<0>(GetParam()); + const PerTileSwizzle per_tile_swizzle(std::get<1>(GetParam())); performTestMultiTensorSwizzle(num_tensors, M, K, rowwise); } @@ -381,6 +389,14 @@ std::vector> multi_tensor_test_cases = { {2, 128, 4352, true}, // Colwise num_tiles_k = 512 / 32 / 4 = 4 selects vec_load_size = 4 (int4 kernel) {2, 512, 4096, false}, + // MXFP8 swizzle: more tensors than one launch takes, several row-wise tiles per CTA, + // and 128-row column-wise chunks + {80, 128, 128, true}, + {80, 128, 128, false}, + {20, 16384, 128, true}, + {20, 128, 8192, false}, + // A full 128-row column-wise chunk and a clipped last one (132 scale rows) + {20, 4224, 2048, false}, }; } // namespace @@ -388,12 +404,14 @@ std::vector> multi_tensor_test_cases = { INSTANTIATE_TEST_SUITE_P( OperatorTest, MultiTensorSwizzleTestSuite, - ::testing::ValuesIn(multi_tensor_test_cases), + ::testing::Combine(::testing::ValuesIn(multi_tensor_test_cases), ::testing::Values(false, true)), [](const testing::TestParamInfo& info) { - return "n" + std::to_string(std::get<0>(info.param)) + - "_M" + std::to_string(std::get<1>(info.param)) + - "_K" + std::to_string(std::get<2>(info.param)) + - (std::get<3>(info.param) ? "_row" : "_col"); + const auto& test_case = std::get<0>(info.param); + return "n" + std::to_string(std::get<0>(test_case)) + + "_M" + std::to_string(std::get<1>(test_case)) + + "_K" + std::to_string(std::get<2>(test_case)) + + (std::get<3>(test_case) ? "_row" : "_col") + + (std::get<1>(info.param) ? "_per_tile" : ""); }); INSTANTIATE_TEST_SUITE_P( diff --git a/tests/cpp/operator/test_normalization_mxfp8.cu b/tests/cpp/operator/test_normalization_mxfp8.cu index 10b33f8e2a1..84c244ff1ff 100644 --- a/tests/cpp/operator/test_normalization_mxfp8.cu +++ b/tests/cpp/operator/test_normalization_mxfp8.cu @@ -102,7 +102,8 @@ void dequantize_2x(Tensor& input, Tensor& output, bool is_training) } template -void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, NormType norm_type, bool is_training, const bool zero_centered_gamma_in_weight_dtype) { +void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, NormType norm_type, bool is_training, const bool zero_centered_gamma_in_weight_dtype, + const bool use_cudnn) { cudaDeviceProp prop; cudaGetDeviceProperties(&prop, 0); @@ -134,6 +135,9 @@ void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, } else { nvte_enable_zero_centered_gamma_in_weight_dtype(false); } + // RMSNorm shapes that are multiples of 128 use Transformer Engine's fused MXFP8 kernel + // unless cuDNN is requested. + nvte_enable_cudnn_norm_fwd(use_cudnn); // Forward kernel float epsilon = 1e-5; @@ -163,6 +167,7 @@ void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, if (zero_centered_gamma_in_weight_dtype) { nvte_enable_zero_centered_gamma_in_weight_dtype(false); } + nvte_enable_cudnn_norm_fwd(false); Tensor dequantized_output("dequantized_output", std::vector{ N, H }, DType::kFloat32, true, true); @@ -197,7 +202,7 @@ void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, nullptr, // amax 1.f, // scale zero_centered_gamma, - true, // CuDNN is the only MXFP8 backend currently + true, // The MXFP8 backends add 1 to a zero-centered gamma as cuDNN does zero_centered_gamma_in_weight_dtype); cudaDeviceSynchronize(); @@ -235,6 +240,95 @@ std::vector> test_cases = { {2048, 12288}, }; +// Index of compact scaling factor (i, j) in the GEMM-swizzled layout: 128x4 tiles of i x j, +// num_tiles_j tiles along j. +size_t swizzled_scale_idx(size_t i, size_t j, size_t num_tiles_j) { + return ((i / 128) * num_tiles_j + j / 4) * 512 + (i % 32) * 16 + ((i % 128) / 32) * 4 + j % 4; +} + +// RMSNorm with MXFP8 output written with GEMM-swizzled scaling factors must give the same +// data as with compact scaling factors, and the compact scaling factors in swizzled order. +// The swizzled output is computed on a single SM, as with an SM margin, which launches the +// kernels over chunks of rows. +template +void performSwizzledScalesTest(const size_t N, const size_t H, const bool is_training) { + if (getDeviceComputeCapability() < blackwellComputeCapability) { + GTEST_SKIP(); + } + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + + const DType itype = TypeInfo::dtype; + const DType otype = TypeInfo::dtype; + Tensor input("input", std::vector{ N, H }, itype); + Tensor gamma("gamma", std::vector{ H }, itype); + Tensor rsigma("rsigma", std::vector{ N }, DType::kFloat32); + Tensor rsigma_swizzled("rsigma_swizzled", std::vector{ N }, DType::kFloat32); + Tensor z("z", std::vector{ N, H }, otype, true, is_training, NVTE_MXFP8_1D_SCALING); + Tensor z_swizzled("z_swizzled", std::vector{ N, H }, otype, true, is_training, + NVTE_MXFP8_1D_SCALING); + z_swizzled.set_with_gemm_swizzled_scales(true); + fillUniform(&input); + fillUniform(&gamma); + + const auto run = [&](Tensor &out, Tensor &rs, const int sm_count) { + Tensor workspace; + nvte_rmsnorm_fwd(input.data(), gamma.data(), 1e-5f, out.data(), rs.data(), + workspace.data(), sm_count, false, 0); + workspace = Tensor("workspace", workspace.rowwise_shape(), workspace.dtype()); + nvte_rmsnorm_fwd(input.data(), gamma.data(), 1e-5f, out.data(), rs.data(), + workspace.data(), sm_count, false, 0); + }; + run(z, rsigma, prop.multiProcessorCount); + run(z_swizzled, rsigma_swizzled, 1); + cudaDeviceSynchronize(); + auto err = cudaGetLastError(); + ASSERT_EQ(err, cudaSuccess) << cudaGetErrorString(err); + z.to_cpu(); + z_swizzled.to_cpu(); + rsigma.to_cpu(); + compareResults("rsigma", rsigma_swizzled, rsigma.rowwise_cpu_dptr(), true, 0, 0); + + const auto bytes = [](const OutputType *p) { return reinterpret_cast(p); }; + compareResults("rowwise_data", bytes(z_swizzled.rowwise_cpu_dptr()), + bytes(z.rowwise_cpu_dptr()), N * H); + const NVTEShape row_shape = z.rowwise_scale_inv_shape(); + const size_t row_dim0 = row_shape.data[0], row_dim1 = row_shape.data[1]; + std::vector ref(row_dim0 * row_dim1); + const uint8_t *compact = z.rowwise_cpu_scale_inv_ptr(); + for (size_t i = 0; i < row_dim0; ++i) { + for (size_t j = 0; j < row_dim1; ++j) { + ref[swizzled_scale_idx(i, j, row_dim1 / 4)] = compact[i * row_dim1 + j]; + } + } + compareResults("rowwise_scales", z_swizzled.rowwise_cpu_scale_inv_ptr(), ref.data(), + ref.size()); + + if (is_training) { + compareResults("colwise_data", bytes(z_swizzled.columnwise_cpu_dptr()), + bytes(z.columnwise_cpu_dptr()), N * H); + // Compact column-wise scaling factors are [rows / 32, cols]; the swizzled layout tiles + // columns by 128 and scale rows by 4. + const NVTEShape col_shape = z.columnwise_scale_inv_shape(); + const size_t col_dim0 = col_shape.data[0], col_dim1 = col_shape.data[1]; + ref.assign(col_dim0 * col_dim1, 0); + compact = z.columnwise_cpu_scale_inv_ptr(); + for (size_t i = 0; i < col_dim0; ++i) { + for (size_t j = 0; j < col_dim1; ++j) { + ref[swizzled_scale_idx(j, i, col_dim0 / 4)] = compact[i * col_dim1 + j]; + } + } + compareResults("colwise_scales", z_swizzled.columnwise_cpu_scale_inv_ptr(), + ref.data(), ref.size()); + } +} + +std::vector> swizzled_scales_test_cases = { + {128, 128}, + {768, 2304}, + {256, 7168}, +}; + std::vector norms = { NormType::LayerNorm, NormType::RMSNorm @@ -246,7 +340,7 @@ class MxNormTestSuite : public ::testing::TestWithParam< std::tuple, - bool, bool, bool>> {}; + bool, bool, bool, bool>> {}; TEST_P(MxNormTestSuite, TestMxNorm) { using namespace transformer_engine; @@ -259,10 +353,15 @@ TEST_P(MxNormTestSuite, TestMxNorm) { const bool zero_centered_gamma = std::get<4>(GetParam()); const bool is_training = std::get<5>(GetParam()); const bool zero_centered_gamma_in_weight_dtype = std::get<6>(GetParam()); + const bool use_cudnn = std::get<7>(GetParam()); + if (norm_type == NormType::LayerNorm && use_cudnn) { + GTEST_SKIP() << "LayerNorm with MXFP8 output always uses cuDNN"; + } TRANSFORMER_ENGINE_TYPE_SWITCH_FP16_FP32_ONLY(input_type, InputType, TRANSFORMER_ENGINE_TYPE_SWITCH_FP8_ONLY(output_type, OutputType, - performTest(size.first, size.second, zero_centered_gamma, norm_type, is_training, zero_centered_gamma_in_weight_dtype); + performTest(size.first, size.second, zero_centered_gamma, norm_type, is_training, zero_centered_gamma_in_weight_dtype, + use_cudnn); ); ); } @@ -277,7 +376,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(test_cases), ::testing::Values(true, false), ::testing::Values(true, false), - ::testing::Values(true, false)), + ::testing::Values(true, false), + ::testing::Values(false, true)), [](const testing::TestParamInfo& info) { std::string name = normToString.at(std::get<0>(info.param)) + "_" + test::typeName(std::get<1>(info.param)) + "X" + @@ -286,6 +386,44 @@ INSTANTIATE_TEST_SUITE_P( std::to_string(std::get<3>(info.param).second) + "X" + std::to_string(std::get<4>(info.param)) + "out" + std::to_string(int(std::get<5>(info.param)) + 1) + "x" + - std::to_string(std::get<6>(info.param)); + std::to_string(std::get<6>(info.param)) + + (std::get<7>(info.param) ? "Xcudnn" : ""); return name; }); + +class MxNormSwizzledScalesTestSuite + : public ::testing::TestWithParam, bool>> {}; + +TEST_P(MxNormSwizzledScalesTestSuite, TestMxRMSNormSwizzledScales) { + using namespace transformer_engine; + using namespace test; + + const DType input_type = std::get<0>(GetParam()); + const DType output_type = std::get<1>(GetParam()); + const auto size = std::get<2>(GetParam()); + const bool is_training = std::get<3>(GetParam()); + + TRANSFORMER_ENGINE_TYPE_SWITCH_FP16_FP32_ONLY(input_type, InputType, + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8_ONLY(output_type, OutputType, + performSwizzledScalesTest(size.first, size.second, is_training); + ); + ); +} + +INSTANTIATE_TEST_SUITE_P( + OperatorTest, + MxNormSwizzledScalesTestSuite, + ::testing::Combine( + ::testing::Values(DType::kFloat32, DType::kBFloat16, DType::kFloat16), + ::testing::Values(DType::kFloat8E5M2, DType::kFloat8E4M3), + ::testing::ValuesIn(swizzled_scales_test_cases), + ::testing::Values(true, false)), + [](const testing::TestParamInfo& info) { + return test::typeName(std::get<0>(info.param)) + "X" + + test::typeName(std::get<1>(info.param)) + "X" + + std::to_string(std::get<2>(info.param).first) + "X" + + std::to_string(std::get<2>(info.param).second) + "X" + + std::to_string(int(std::get<3>(info.param)) + 1) + "x"; + }); diff --git a/tests/cpp/operator/test_swizzle.cu b/tests/cpp/operator/test_swizzle.cu index 8990ce8db13..a442215ae15 100644 --- a/tests/cpp/operator/test_swizzle.cu +++ b/tests/cpp/operator/test_swizzle.cu @@ -308,7 +308,17 @@ void performTestGroupedSwizzleMXFP8(const int num_tensors, const size_t M, const num_tensors * col_numel); } -class SwizzleTestSuite : public ::testing::TestWithParam, std::pair, bool>> {}; +// Sets NVTE_MXFP8_PER_TILE_SWIZZLE while it lives, so that MXFP8 swizzles use the per-tile +// kernels (enable) or the default ones. +class PerTileSwizzle { + public: + explicit PerTileSwizzle(bool enable) { + setenv("NVTE_MXFP8_PER_TILE_SWIZZLE", enable ? "1" : "0", 1); + } + ~PerTileSwizzle() { unsetenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } +}; + +class SwizzleTestSuite : public ::testing::TestWithParam, std::pair, bool, bool>> {}; TEST_P(SwizzleTestSuite, TestSwizzle) { @@ -318,6 +328,7 @@ TEST_P(SwizzleTestSuite, TestSwizzle) { const auto num_tiles = std::get<0>(GetParam()); const auto scaling_mode = std::get<1>(GetParam()); const auto transa = std::get<2>(GetParam()); + const PerTileSwizzle per_tile_swizzle(std::get<3>(GetParam())); performTestSwizzle1D(num_tiles.first, num_tiles.second, scaling_mode.first, scaling_mode.second, @@ -1004,6 +1015,7 @@ std::vector> num_tiles = { {8, 1}, // narrow_m with 8 M-tiles {31, 1}, // narrow_m at boundary (31 < TB_DIM=32) {1, 31}, // narrow_k at boundary (31 < TB_DIM=32) + {1600, 1}, // MXFP8: more row-wise tiles than one CTA per tile allows }; // Raw {M, K} data shapes for unswizzle tests. Includes aligned cases (scale dims @@ -1039,7 +1051,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( ::testing::ValuesIn(num_tiles), ::testing::ValuesIn(scaling_mode), - ::testing::ValuesIn(transa) + ::testing::ValuesIn(transa), + ::testing::Values(false, true) ), [](const testing::TestParamInfo& info) { std::string name = "ntiles" + @@ -1047,7 +1060,8 @@ INSTANTIATE_TEST_SUITE_P( std::to_string(std::get<0>(info.param).second) + "smode" + std::to_string(std::get<1>(info.param).first) + "X"+ std::to_string(std::get<1>(info.param).second) + "trans" + - std::to_string(std::get<2>(info.param)); + std::to_string(std::get<2>(info.param)) + + (std::get<3>(info.param) ? "PerTile" : ""); return name; }); diff --git a/tests/cpp_distributed/CMakeLists.txt b/tests/cpp_distributed/CMakeLists.txt index 9ca071edfaf..2dd7b301c14 100644 --- a/tests/cpp_distributed/CMakeLists.txt +++ b/tests/cpp_distributed/CMakeLists.txt @@ -56,6 +56,10 @@ find_package(CUDAToolkit REQUIRED) find_package(OpenMP REQUIRED) find_package(MPI REQUIRED) +option(NVTE_WITH_CUTEDSL + "Link the tests against libpython, required when libtransformer_engine.so was built with NVTE_WITH_CUTEDSL=ON" + "$ENV{NVTE_WITH_CUTEDSL}") + # -- NCCL core ---------------------------------------------------------------- # Anchor on libnccl and derive nccl.h from the same install prefix so the # header and library can't drift across installs. @@ -127,6 +131,19 @@ add_executable(test_ep test_ep.cu ../cpp/test_common.cu) target_include_directories(test_ep PRIVATE ${EP_TEST_COMMON_INCLUDES}) target_link_libraries(test_ep PUBLIC ${EP_TEST_COMMON_LIBS}) +if(NVTE_WITH_CUTEDSL) + # Executables that link libtransformerengine.so must also link to the Python::Python to provide + # the definitions of these CPython symbols at runtime via dynamic linking. + # Find the interpreter too so older CMake versions use its version to locate libpython. + find_package(Python COMPONENTS Interpreter Development.Embed REQUIRED) + foreach(test_target test_comm_gemm test_ep) + target_link_libraries(${test_target} PRIVATE + "-Wl,--no-as-needed" + Python::Python + "-Wl,--as-needed") + endforeach() +endif() + # Do NOT use gtest_discover_tests - these binaries require multi-process # launch via run_test_ep.sh, not direct single-process execution. message(STATUS "EP distributed tests enabled (NCCL EP loaded from libnccl_ep.so at runtime)") diff --git a/tests/pytorch/attention/test_attention_backend_selection.py b/tests/pytorch/attention/test_attention_backend_selection.py index c2944eece3b..bd587416a79 100644 --- a/tests/pytorch/attention/test_attention_backend_selection.py +++ b/tests/pytorch/attention/test_attention_backend_selection.py @@ -12,6 +12,7 @@ from transformer_engine.pytorch import DotProductAttention from transformer_engine.pytorch.attention.dot_product_attention import ( + backends as dpa_backends, dot_product_attention as dpa_module, utils as dpa_utils, ) @@ -42,19 +43,20 @@ def isolated_backends(monkeypatch): @pytest.mark.parametrize( - "capability,training,backward,deterministic,unfused,expect_flash", + "capability,training,backward,deterministic,head_dims,unfused,expect_flash", [ - pytest.param((12, 0), True, True, True, True, False, id="sm120-training"), - pytest.param((12, 0), False, True, True, True, False, id="sm120-eval-backward"), - pytest.param((12, 1), False, True, True, True, False, id="sm121-eval-backward"), - pytest.param((12, 0), False, False, True, True, True, id="forward-only"), - pytest.param((12, 0), True, True, False, True, True, id="nondeterministic"), - pytest.param((12, 0), True, True, True, False, False, id="no-fallback"), - pytest.param((10, 0), True, True, True, True, True, id="sm100-unaffected"), + pytest.param((12, 0), True, True, True, (128, 128), True, False, id="sm120-training"), + pytest.param((12, 0), False, True, True, (128, 128), True, False, id="sm120-eval-backward"), + pytest.param((12, 1), False, True, True, (128, 128), True, False, id="sm121-eval-backward"), + pytest.param((12, 0), False, False, True, (128, 128), True, True, id="forward-only"), + pytest.param((12, 0), True, True, False, (128, 128), True, True, id="nondeterministic"), + pytest.param((12, 0), True, True, True, (128, 128), False, False, id="no-fallback"), + pytest.param((10, 0), True, True, True, (128, 128), True, True, id="sm100-d128"), + pytest.param((10, 0), True, True, True, (256, 256), True, False, id="sm100-d256"), ], ) def test_fa4_deterministic_backend_selection( - monkeypatch, capability, training, backward, deterministic, unfused, expect_flash + monkeypatch, capability, training, backward, deterministic, head_dims, unfused, expect_flash ): monkeypatch.setenv("NVTE_UNFUSED_ATTN", str(int(unfused))) monkeypatch.setattr(dpa_utils, "get_device_compute_capability", lambda device: capability) @@ -66,8 +68,8 @@ def test_fa4_deterministic_backend_selection( device=torch.device("cuda:0"), qkv_dtype=torch.bfloat16, qkv_layout="bshd_bshd_bshd", - head_dim_qk=128, - head_dim_v=128, + head_dim_qk=head_dims[0], + head_dim_v=head_dims[1], attn_mask_type="causal", window_size=(-1, 0), deterministic=deterministic, @@ -82,6 +84,44 @@ def test_fa4_deterministic_backend_selection( assert available == [expect_flash, False, unfused] +@pytest.mark.parametrize( + "version,backward,expect_flash", + [ + pytest.param("4.0.0b30", False, False, id="b30-forward"), + pytest.param("4.0.0b31", False, True, id="b31-forward"), + pytest.param("4.0.0b32", True, False, id="b32-backward"), + pytest.param("4.0.0b33", True, True, id="b33-backward"), + ], +) +@pytest.mark.parametrize("path", ["compact", "padding", "all-gather"]) +def test_fa4_d256_seqused_backend_selection(monkeypatch, version, backward, expect_flash, path): + """FA4 D=256 eligibility follows seqused_q/k forward/backward support.""" + monkeypatch.setenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "1") + monkeypatch.setattr(dpa_utils, "get_device_compute_capability", lambda device: (10, 0)) + monkeypatch.setattr(dpa_utils, "get_cudnn_version", lambda: (9, 26, 0)) + monkeypatch.setattr(dpa_utils.FlashAttentionUtils, "v4_is_installed", True) + monkeypatch.setattr(dpa_utils.FlashAttentionUtils, "fa4_version", PkgVersion(version)) + monkeypatch.setattr(dpa_utils.FlashAttentionUtils, "v4_validate_head_dims", None) + params = dpa_utils.AttentionParams( + device=torch.device("cuda:0"), + qkv_dtype=torch.bfloat16, + qkv_layout="thd_thd_thd", + head_dim_qk=256, + head_dim_v=256, + attn_mask_type="padding_causal", + pad_between_seqs=path == "padding", + context_parallel=path == "all-gather", + cp_comm_type="all_gather" if path == "all-gather" else "p2p", + deterministic=False, + requires_backward=backward, + ) + flash, selected_version, _, _, _, available = dpa_utils.get_attention_backend(params) + expected = path == "compact" or expect_flash + assert bool(flash) is expected + assert selected_version == (dpa_utils.FlashAttentionUtils.fa4_version if expected else None) + assert bool(available[0]) is expected + + @pytest.mark.parametrize("changed_field", ["device", "requires_backward"]) def test_thd_backend_cache_tracks_execution_context(monkeypatch, changed_field): selector = Mock(return_value=(False, None, False, None, True, [False, False, True])) @@ -161,3 +201,35 @@ def test_fa4_cached_selection_from_other_device(monkeypatch): assert selector.call_args.args[0].device == qkv[0].device assert dpa_module._attention_backends["use_unfused_attention"] assert all(t.grad is not None and torch.isfinite(t.grad).all() for t in qkv) + + +def _fa4_normalized_kwargs(kwargs): + received = {} + dpa_backends._fa4_with_none_window_sentinel(received.update)(**kwargs) + return received + + +@pytest.mark.parametrize( + "sent,expected", + [ + ({"window_size": (-1, 0)}, {"window_size": (None, 0)}), + ({"window_size": (511, -1)}, {"window_size": (511, None)}), + # Other negative offsets are not the -1 sentinel. + ({"window_size": (7, -8)}, {"window_size": (7, -8)}), + ( + {"window_size_left": -1, "window_size_right": 0}, + {"window_size_left": None, "window_size_right": 0}, + ), + ( + {"window_size_left": 0, "window_size_right": -1}, + {"window_size_left": 0, "window_size_right": None}, + ), + ( + {"window_size_left": 7, "window_size_right": -8}, + {"window_size_left": 7, "window_size_right": -8}, + ), + ], +) +def test_fa4_window_sentinel_normalization(sent, expected): + """Convert only TE's -1 sentinel for both FA4 window argument forms.""" + assert _fa4_normalized_kwargs(sent) == expected diff --git a/tests/pytorch/attention/test_attention_with_cp.py b/tests/pytorch/attention/test_attention_with_cp.py index e4b5ad86edb..56055ca1694 100644 --- a/tests/pytorch/attention/test_attention_with_cp.py +++ b/tests/pytorch/attention/test_attention_with_cp.py @@ -12,6 +12,7 @@ import pathlib import logging import copy +from collections.abc import Callable from collections import deque import pytest import torch @@ -60,6 +61,7 @@ "cp_2_1": ModelConfig(2, 4096, 12, 128, num_gqa_groups=2), # GQA "cp_2_2": ModelConfig(2, 4096, 32, 128, attn_mask_type="causal", window_size=(128, 0)), # GQA "cp_2_3": ModelConfig(2, 4096, 12, 128, num_gqa_groups=2, window_size=(512, 512)), # GQA + "cp_2_4": ModelConfig(2, 4096, 32, 256, num_gqa_groups=4, attn_mask_type="causal"), # GQA "cp_3_0": ModelConfig(2, 4096, 128, 192, attn_mask_type="causal", head_dim_v=128), # MLA "cp_3_1": ModelConfig(2, 4096, 12, 192, head_dim_v=128), # MLA "cp_3_2": ModelConfig( @@ -104,8 +106,9 @@ class PoolWorker: _STDERR_BUFFER_LINES = 200 # ring cap (~40 KB ceiling) _STDERR_TAIL_CHARS = 4000 # how much to attach to the AssertionError - def __init__(self, world_size: int): + def __init__(self, world_size: int, reset_all_pools: Callable[[], None]): self.world_size = world_size + self._reset_all_pools = reset_all_pools self.proc: subprocess.Popen | None = None self._stderr_buf: deque[str] = deque(maxlen=self._STDERR_BUFFER_LINES) @@ -221,10 +224,13 @@ def submit(self, kwargs: dict, timeout: float = POOL_SUBMIT_TIMEOUT_SEC) -> None sys.stderr.flush() raise first_err = e + # Pools with different world sizes can occupy the same GPUs. Reset + # all pools so the retry starts without leftover CUDA/NCCL state. + self._reset_all_pools() sys.stderr.write( f"[POOL-RETRY] status=retrying test={test_id!r} " f"world_size={self.world_size} attempt={attempt + 1} " - f"error={msg_head!r}; respawning pool and retrying\n" + f"error={msg_head!r}; reset all pools and retrying\n" ) sys.stderr.flush() raise first_err # unreachable; loop either returns or raises @@ -297,11 +303,15 @@ def cp_pool(): """Returns a callable: cp_pool(world_size) -> PoolWorker.""" pools: dict[int, PoolWorker] = {} + def _reset_all_pools() -> None: + for pool in pools.values(): + pool.shutdown() + def _get(world_size: int) -> PoolWorker: if world_size > torch.cuda.device_count(): pytest.skip(f"Test requires {world_size} GPUs, but found {torch.cuda.device_count()}") if world_size not in pools: - pools[world_size] = PoolWorker(world_size) + pools[world_size] = PoolWorker(world_size, _reset_all_pools) return pools[world_size] yield _get @@ -319,9 +329,10 @@ def _submit(pool: PoolWorker, **kwargs) -> None: dtypes = ["bf16", "fp16"] qkv_formats = ["bshd", "sbhd", "thd"] cp_comm_types = ["p2p", "all_gather", "a2a", "a2a+p2p"] +essential_model_configs_flash_attn = model_configs_flash_attn if test_essential: configs = ["cp_2_0", "cp_2_2", "cp_3_0", "cp_3_3"] - model_configs_flash_attn = {k: model_configs_flash_attn[k] for k in configs} + essential_model_configs_flash_attn = {k: model_configs_flash_attn[k] for k in configs} dtypes = ["bf16"] qkv_formats = ["sbhd", "thd"] @@ -336,7 +347,7 @@ def _submit(pool: PoolWorker, **kwargs) -> None: ) @pytest.mark.skipif(get_device_compute_capability() < (8, 0), reason="CP tests require sm80+.") @pytest.mark.parametrize("dtype", dtypes) -@pytest.mark.parametrize("model", model_configs_flash_attn.keys()) +@pytest.mark.parametrize("model", essential_model_configs_flash_attn.keys()) @pytest.mark.parametrize("qkv_format", qkv_formats) @pytest.mark.parametrize("cp_comm_type", cp_comm_types) @pytest.mark.parametrize("pad_between_seqs", [False, True]) @@ -552,6 +563,7 @@ def test_cp_with_flash_attention_softcap(cp_pool, cp_comm_type): dtypes = ["bf16", "fp16", "fp8"] qkv_formats = ["bshd", "sbhd", "thd"] cp_comm_types = ["p2p", "all_gather", "a2a", "a2a+p2p"] +essential_model_configs_fused_attn = model_configs_fused_attn if test_essential: configs = [ "cp_1_0", @@ -567,14 +579,14 @@ def test_cp_with_flash_attention_softcap(cp_pool, cp_comm_type): "cp_5_0", "cp_5_1", ] - model_configs_fused_attn = {k: model_configs_fused_attn[k] for k in configs} + essential_model_configs_fused_attn = {k: model_configs_fused_attn[k] for k in configs} dtypes = ["bf16", "fp8"] qkv_formats = ["sbhd", "thd"] @pytest.mark.skipif(get_device_compute_capability() < (8, 0), reason="CP tests require sm80+.") @pytest.mark.parametrize("dtype", dtypes) -@pytest.mark.parametrize("model", model_configs_fused_attn.keys()) +@pytest.mark.parametrize("model", essential_model_configs_fused_attn.keys()) @pytest.mark.parametrize("qkv_format", qkv_formats) @pytest.mark.parametrize("cp_comm_type", cp_comm_types) @pytest.mark.parametrize("fp8_bwd", [True, False]) @@ -779,13 +791,22 @@ def test_cp_with_fused_attention_no_load_balance(cp_pool): ) -def test_cp_with_flash_attention_no_load_balance(cp_pool): +@pytest.mark.parametrize("model", ("cp_2_0", "cp_2_4")) +def test_cp_with_flash_attention_no_load_balance(cp_pool, model): """Check the supported unpadded FlashAttention path.""" - config = copy.deepcopy(model_configs_flash_attn["cp_2_0"]) + config = copy.deepcopy(model_configs_flash_attn[model]) + fa4_expected = ( + config.head_dim_qk == 256 + and (10, 0) <= get_device_compute_capability() < (12, 0) + and fa4_enabled + and FlashAttentionUtils.v4_is_installed + and FlashAttentionUtils.fa4_version >= FlashAttentionUtils.v4_0_0_beta33 + and not _deterministic + ) config.context_parallel = True config.cp_comm_type = "all_gather" config.attn_mask_type = "padding_causal" - available_backends, _, _ = get_available_attention_backends( + available_backends, flash_backend, _ = get_available_attention_backends( config, qkv_dtype=torch.bfloat16, qkv_layout="thd_thd_thd", @@ -793,12 +814,16 @@ def test_cp_with_flash_attention_no_load_balance(cp_pool): is_training=True, deterministic=_deterministic, ) + if fa4_expected: + assert available_backends[0] and flash_backend is not None and flash_backend.major == 4 + elif config.head_dim_qk == 256: + pytest.skip("D=256 THD all-gather requires SM100/SM110 FA4 b33+ non-deterministic.") if not available_backends[0]: pytest.skip("FlashAttention is unavailable.") _submit( cp_pool(2), dtype="bf16", - model="cp_2_0", + model=model, qkv_format="thd", kernel_backend="FlashAttention", cp_comm_type="all_gather", diff --git a/tests/pytorch/attention/test_fused_mla_q_uproj.py b/tests/pytorch/attention/test_fused_mla.py similarity index 61% rename from tests/pytorch/attention/test_fused_mla_q_uproj.py rename to tests/pytorch/attention/test_fused_mla.py index 421c849a6ed..1fcd9a8e218 100644 --- a/tests/pytorch/attention/test_fused_mla_q_uproj.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -2,24 +2,39 @@ # # See LICENSE for license information. -"""Unit tests for FusedMLAQUpProjRopeQuant. +"""Unit tests for the fused multi-latent attention (MLA) kernels. + +- FusedMLAQUpProjRopeQuant: query up-projection, RoPE and MXFP8 quantization (cuDNN FE). +- fused_mla_kv_rope_mxfp8: key/value split, key RoPE and MXFP8 quantization. The reference + applies Megatron-LM's MLA key/value RoPE (the Triton kernel in mla_rope_utils) and quantizes + key and value with mxfp8_quantize_only; the fused kernel must produce the same bytes. Run: - pytest tests/pytorch/attention/test_fused_mla_q_uproj.py -v + pytest tests/pytorch/attention/test_fused_mla.py -v """ import os +import pathlib +import sys import pytest import torch -import transformer_engine.pytorch # registers transformer_engine_torch +import transformer_engine.pytorch as te import transformer_engine_torch as tex from transformer_engine.pytorch.attention import FusedMLAQUpProjFunction, FusedMLAQUpProjRopeQuant +from transformer_engine.pytorch.attention.dot_product_attention.utils import ( + fused_mla_kv_rope_mxfp8, + mxfp8_quantize_only, +) from transformer_engine.pytorch.cpp_extensions import general_gemm as _fused_general_gemm from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor -# All tests in this file require GB200/SM100 hardware (cuDNN FE fused kernel). +_current_file = pathlib.Path(__file__).resolve() +sys.path = [str(_current_file.parent)] + sys.path +from mla_rope_utils import HAVE_TRITON, apply_mla_rope_kv, build_rope_tables + +# All tests in this file require GB200/SM100 hardware. pytestmark = pytest.mark.launch_on_gb200 # DSv3 671B MLA dims @@ -307,3 +322,143 @@ def test_fused_mla_q_uproj_autograd() -> None: torch.testing.assert_close(x.grad.reshape(tokens, Q_LORA_RANK), grad_x_ref, atol=0.5, rtol=0.1) torch.testing.assert_close(gamma.grad, grad_gamma_ref, atol=0.5, rtol=0.1) torch.testing.assert_close(w_bf16.grad, grad_w_ref, atol=0.5, rtol=0.1) + + +# --- fused_mla_kv_rope_mxfp8 --------------------------------------------------------------------- + +mxfp8_available, reason_for_no_mxfp8 = te.is_mxfp8_available(return_reason=True) +sm100 = torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10 +if not mxfp8_available: + _kv_rope_skip_reason = reason_for_no_mxfp8 +elif not sm100: + _kv_rope_skip_reason = "The fused kernel requires SM 10.0+." +elif not HAVE_TRITON: + _kv_rope_skip_reason = "The reference MLA RoPE needs Triton." +else: + _kv_rope_skip_reason = "" +requires_kv_rope_kernel = pytest.mark.skipif( + bool(_kv_rope_skip_reason), reason=_kv_rope_skip_reason +) + +HEAD_DIM_V = 128 +KV_LORA_RANK = 512 +MXFP8_BUFFERS = ("_rowwise_data", "_rowwise_scale_inv", "_columnwise_data", "_columnwise_scale_inv") + + +def _kv_quantizer(): + return MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True) + + +def _kv_rope_inputs(s, b, h, cos_dtype, seed=1234): + torch.manual_seed(seed) + kv = (torch.randn(s, b, h, HEAD_DIM_NOPE + HEAD_DIM_V, device="cuda") * 4).to(torch.bfloat16) + # An all-zero scaling block and a value that saturates the scale of its blocks. + kv[5, 0, 0, :32] = 0 + kv[7, 0, h - 1, 3] = 3.0e4 + kv[9, b - 1, 1, HEAD_DIM_NOPE + 17] = -2.5e4 + # k_rope is a slice of the key/value down-projection output, as in MLA. + kv_down = (torch.randn(s, b, KV_LORA_RANK + HEAD_DIM_ROPE, device="cuda") * 4).to( + torch.bfloat16 + ) + k_rope = kv_down[..., KV_LORA_RANK:].unsqueeze(-2) + cos, sin = build_rope_tables(s, emb_dim=HEAD_DIM_ROPE, device="cuda") + # A magnitude scale as in YaRN, so that the products are not exact. + cos, sin = (cos * 1.2).to(cos_dtype), (sin * 1.2).to(cos_dtype) + return kv, k_rope, cos, sin + + +def _kv_rope_reference(kv, k_rope, cos, sin): + key, value = apply_mla_rope_kv(kv, k_rope, cos_table=cos, sin_table=sin) + return mxfp8_quantize_only( + [(key.contiguous(), _kv_quantizer()), (value.contiguous(), _kv_quantizer())], "sbhd" + ) + + +def _assert_identical(actual, expected, name): + assert actual.shape == expected.shape, f"{name}: shape {actual.shape} vs {expected.shape}" + for buffer in MXFP8_BUFFERS: + a, e = getattr(actual, buffer), getattr(expected, buffer) + assert a.shape == e.shape, f"{name}.{buffer}: shape {tuple(a.shape)} vs {tuple(e.shape)}" + a, e = a.view(torch.uint8), e.view(torch.uint8) + mismatches = int((a != e).sum()) + assert mismatches == 0, f"{name}.{buffer}: {mismatches} of {e.numel()} bytes differ" + + +@requires_kv_rope_kernel +@pytest.mark.parametrize( + "s, b, h", [(4096, 1, 128), (2048, 2, 64), (512, 1, 16), (256, 3, 8), (128, 2, 2)] +) +@pytest.mark.parametrize("cos_dtype", [torch.float32, torch.bfloat16], ids=["fp32", "bf16"]) +def test_matches_rope_then_quantize(s, b, h, cos_dtype): + """Same bytes as the RoPE / split kernel followed by MXFP8 quantization.""" + kv, k_rope, cos, sin = _kv_rope_inputs(s, b, h, cos_dtype) + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos, sin) + cos4, sin4 = cos.view(s, 1, 1, HEAD_DIM_ROPE), sin.view(s, 1, 1, HEAD_DIM_ROPE) + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope, cos4, sin4, HEAD_DIM_V) + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + + +@requires_kv_rope_kernel +def test_contiguous_k_rope_and_longer_rope_tables(): + """A contiguous [s, b, emb_dim] k_rope and cos / sin with more rows than tokens.""" + s, b, h = 256, 2, 4 + kv, k_rope, _, _ = _kv_rope_inputs(s, b, h, torch.bfloat16) + cos, sin = build_rope_tables(2 * s, emb_dim=HEAD_DIM_ROPE, device="cuda") + cos, sin = cos.to(torch.bfloat16), sin.to(torch.bfloat16) + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos[:s], sin[:s]) + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope.squeeze(-2).contiguous(), cos, sin, 128) + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + + +@requires_kv_rope_kernel +def test_non_finite_values(): + """Inf and NaN inputs give the quantizer's scaling factors and data.""" + s, b, h = 128, 1, 4 + kv, k_rope, cos, sin = _kv_rope_inputs(s, b, h, torch.bfloat16) + kv[3, 0, 0, 40] = float("inf") + kv[40, 0, 1, HEAD_DIM_NOPE + 5] = float("-inf") + kv[64, 0, 2, 100] = float("nan") + kv[96:128, 0, 3, HEAD_DIM_NOPE + 32 : HEAD_DIM_NOPE + 64] = float("nan") + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos, sin) + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V) + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + + +@requires_kv_rope_kernel +@pytest.mark.parametrize( + "s, h, message", [(96, 4, "multiple of 128"), (128, 3, "must be even")], ids=["s", "h"] +) +def test_unsupported_shapes(s, h, message): + kv, k_rope, cos, sin = _kv_rope_inputs(s, 1, h, torch.bfloat16) + with pytest.raises((RuntimeError, ValueError), match=message): + fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V) + + +@requires_kv_rope_kernel +def test_2d_quantizer_rejected(): + """The kernel quantizes 1D blocks, so 2D MXFP8 quantizers are rejected.""" + kv, k_rope, cos, sin = _kv_rope_inputs(128, 1, 4, torch.bfloat16) + quantizer = MXFP8Quantizer( + fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True, with_2d_quantization=True + ) + with pytest.raises(RuntimeError, match="2D MXFP8 quantization"): + fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V, quantizer, quantizer) + + +@requires_kv_rope_kernel +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Needs two GPUs.") +def test_inputs_on_non_current_device(): + """Outputs are allocated, and the kernel runs, on the inputs' device.""" + with torch.cuda.device(1): + kv, k_rope, cos, sin = _kv_rope_inputs(128, 2, 4, torch.bfloat16) + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos, sin) + assert torch.cuda.current_device() == 0 + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V) + assert key._rowwise_data.device == kv.device and value._rowwise_data.device == kv.device + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + with pytest.raises(RuntimeError, match="same device"): + fused_mla_kv_rope_mxfp8(kv, k_rope.to("cuda:0"), cos, sin, HEAD_DIM_V) diff --git a/tests/pytorch/distributed/test_fusible_ops.py b/tests/pytorch/distributed/test_fusible_ops.py index c314bf5bb69..b4f3bce3a4e 100644 --- a/tests/pytorch/distributed/test_fusible_ops.py +++ b/tests/pytorch/distributed/test_fusible_ops.py @@ -35,7 +35,7 @@ _current_file = pathlib.Path(__file__).resolve() # Prepend so installed packages with a top-level utils module cannot shadow the test helpers. sys.path = [str(_current_file.parent.parent)] + sys.path -from utils import dtype_tols, make_recipe, quantization_tols +from utils import assert_close, dtype_tols, make_recipe, quantization_tols # Check what quantization schemes are supported @@ -331,6 +331,7 @@ def _test_basic_linear( quantized_weight: bool = False, tensor_parallel_mode: str = "column", sequence_parallel: bool = False, + compile_model: bool = False, ) -> None: # Skip invalid configurations @@ -428,6 +429,8 @@ def _test_basic_linear( # Implementation with fusible operation recipe = make_recipe(quantization) + if compile_model and recipe is not None: + recipe.backward_override = None with te.quantized_model_init(enabled=quantized_weight, recipe=recipe): op = te_ops.BasicLinear( in_features, @@ -441,9 +444,17 @@ def _test_basic_linear( with torch.no_grad(): op.weight.copy_(w_test) del w_test - with te.autocast(enabled=quantized_compute, recipe=recipe): - y_test = op(x_test) - y_test.backward(dy_test) + model = te_ops.Sequential(op) + + def forward(x): + if not quantized_compute: + return model(x) + with te.autocast(enabled=quantized_compute, recipe=recipe): + return model(x) + + if compile_model: + torch._dynamo.reset() + forward = torch.compile(forward, fullgraph=True) # Expected numerical error tols = dtype_tols(dtype) @@ -452,13 +463,15 @@ def _test_basic_linear( if quantized_compute: tols = quantization_tols(quantization) - # Check results - y_test = y_test.to(dtype=torch.float64, device="cpu") - dx_test = x_test.grad.to(dtype=torch.float64, device="cpu") - dw_test = op.weight.grad.to(dtype=torch.float64, device="cpu") - torch.testing.assert_close(y_test, y_ref, **tols) - torch.testing.assert_close(dx_test, dx_ref, **tols) - torch.testing.assert_close(dw_test, dw_ref, **tols) + # Repeated calls also exercise the compiled backward and cached graph. + for _ in range(3 if compile_model else 1): + model.zero_grad(set_to_none=True) + x_test.grad = None + y_test = forward(x_test) + y_test.backward(dy_test) + assert_close(y_test, y_ref, **tols) + assert_close(x_test.grad, dx_ref, **tols) + assert_close(op.weight.grad, dw_ref, **tols) def _test_linear( @@ -472,6 +485,7 @@ def _test_linear( quantized_weight: bool = False, tensor_parallel_mode: str = "column", sequence_parallel: bool = False, + compile_model: bool = False, ) -> None: # Skip invalid configurations @@ -594,6 +608,8 @@ def _test_linear( # Implementation with fusible operation recipe = make_recipe(quantization) + if compile_model and recipe is not None: + recipe.backward_override = None with te.quantized_model_init(enabled=quantized_weight, recipe=recipe): model = te_ops.Sequential( te_ops.Linear( @@ -613,9 +629,16 @@ def _test_linear( model[0].bias.copy_(b_test) del w_test del b_test - with te.autocast(enabled=quantized_compute, recipe=recipe): - y_test = model(x_test) - y_test.backward(dy_test) + + def forward(x): + if not quantized_compute: + return model(x) + with te.autocast(enabled=quantized_compute, recipe=recipe): + return model(x) + + if compile_model: + torch._dynamo.reset() + forward = torch.compile(forward, fullgraph=True) # Expected numerical error tols = dtype_tols(dtype) @@ -624,16 +647,16 @@ def _test_linear( if quantized_compute: tols = quantization_tols(quantization) - # Check results - y_test = y_test.to(dtype=torch.float64, device="cpu") - dx_test = x_test.grad.to(dtype=torch.float64, device="cpu") - dw_test = model[0].weight.grad.to(dtype=torch.float64, device="cpu") - torch.testing.assert_close(y_test, y_ref, **tols) - torch.testing.assert_close(dx_test, dx_ref, **tols) - torch.testing.assert_close(dw_test, dw_ref, **tols) - if bias: - db_test = model[0].bias.grad.to(dtype=torch.float64, device="cpu") - torch.testing.assert_close(db_test, db_ref, **tols) + for _ in range(3 if compile_model else 1): + model.zero_grad(set_to_none=True) + x_test.grad = None + y_test = forward(x_test) + y_test.backward(dy_test) + assert_close(y_test, y_ref, **tols) + assert_close(x_test.grad, dx_ref, **tols) + assert_close(model[0].weight.grad, dw_ref, **tols) + if bias: + assert_close(model[0].bias.grad, db_ref, **tols) def _test_mlp( @@ -1034,6 +1057,35 @@ def run_parallel_tests() -> None: _test_fp8_scale_update() +def run_compile_parallel_tests() -> None: + """Compile the existing TP numerical test, including the Linear+Bias fusion.""" + quantizations = [None, "fp8_current_scaling"] if fp8_available else [None] + for quantization, mode, sp in itertools.product( + quantizations, ("column", "row"), (False, True) + ): + if torch.distributed.get_rank(world_group()) == 0: + print(f"Compile BasicLinear: {quantization=}, {mode=}, {sp=}", flush=True) + _test_basic_linear( + dtype=torch.bfloat16, + quantization=quantization, + tensor_parallel_mode=mode, + sequence_parallel=sp, + compile_model=True, + ) + # Row-parallel Linear uses standalone collectives without compile support. + if mode == "column": + if torch.distributed.get_rank(world_group()) == 0: + print(f"Compile Linear: {quantization=}, {mode=}, {sp=}", flush=True) + _test_linear( + bias=True, + dtype=torch.bfloat16, + quantization=quantization, + tensor_parallel_mode=mode, + sequence_parallel=sp, + compile_model=True, + ) + + # Parallel job sizes _world_sizes = [torch.cuda.device_count()] if 1 not in _world_sizes: @@ -1043,7 +1095,8 @@ def run_parallel_tests() -> None: @pytest.mark.parametrize("world_size", _world_sizes) -def test_distributed_fuser_ops(world_size: int) -> None: +@pytest.mark.parametrize("compile_model", [False, True], ids=["eager", "compile"]) +def test_distributed_fuser_ops(world_size: int, compile_model: bool) -> None: """Launch parallel job that runs parallel tests""" python_exe = pathlib.Path(sys.executable) current_file = pathlib.Path(__file__).resolve() @@ -1055,6 +1108,8 @@ def test_distributed_fuser_ops(world_size: int) -> None: current_file, "--parallel", ] + if compile_model: + command.append("--compile") with tempfile.TemporaryDirectory(prefix="te-test-fusible-ops-") as temp_dir: env = dict(os.environ) env["NVTE_TEST_RDZV_PATH"] = str(pathlib.Path(temp_dir) / "rdzv") @@ -1064,9 +1119,21 @@ def test_distributed_fuser_ops(world_size: int) -> None: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--parallel", action="store_true", help="Run parallel tests") + parser.add_argument("--compile", action="store_true", help="Use the compiled TP test suite") args = parser.parse_args() + if args.compile and not args.parallel: + parser.error("--compile requires --parallel") if args.parallel: - run_parallel_tests() + try: + if args.compile: + run_compile_parallel_tests() + else: + run_parallel_tests() + torch.distributed.barrier(world_group()) + torch.cuda.synchronize() + finally: + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() if __name__ == "__main__": diff --git a/tests/pytorch/distributed/test_parallel_cross_entropy.py b/tests/pytorch/distributed/test_parallel_cross_entropy.py index 16323154939..a0dac8ca15e 100644 --- a/tests/pytorch/distributed/test_parallel_cross_entropy.py +++ b/tests/pytorch/distributed/test_parallel_cross_entropy.py @@ -29,6 +29,7 @@ def _run_tensor_parallel(rank, world_size, init_file, label_smoothing): init_method=f"file://{init_file}", rank=rank, world_size=world_size, + device_id=device, ) try: generator = torch.Generator().manual_seed(2025) @@ -88,6 +89,69 @@ def _run_tensor_parallel(rank, world_size, init_file, label_smoothing): dist.destroy_process_group() +def _run_tensor_parallel_argmax(rank, world_size, init_file, label_smoothing): + """Two-rank worker: the argmax over vocabulary shards, with and without multiple blocks.""" + + torch.cuda.set_device(rank) + device = torch.device("cuda", rank) + dist.init_process_group( + backend="nccl", + init_method=f"file://{init_file}", + rank=rank, + world_size=world_size, + ) + try: + generator = torch.Generator().manual_seed(2026) + for shape in ((2, 3, 22), (1, 4, 80000)): + vocab = shape[-1] + local_vocab = vocab // world_size + target = torch.randint(0, vocab, shape[:-1], generator=generator).to(device) + target[0, 2] = -100 + values = torch.randn(shape, generator=generator).to(device) + peak = 8.0 + values[0, 0, [3, local_vocab + 4]] = peak # equal maxima on both ranks + values[0, 1, [local_vocab + 2, vocab - 2]] = peak # equal maxima on rank 1 + values[0, 2, vocab - 1] = peak # the maximum on the last rank only + if shape[0] > 1: + # NaN counts as the maximum, as in torch.argmax: the first NaN wins. + values[1, 0, [5, local_vocab + 1]] = float("nan") # NaN on both ranks + values[1, 1, 2] = peak + values[1, 1, local_vocab + 3] = float("nan") # NaN after a larger rank-0 maximum + values[-1, -1] = 0.5 # a constant row + vocab_start = rank * local_vocab + for dtype in (torch.float32, torch.bfloat16): + global_values = values.to(dtype) + expected_argmax = torch.argmax(global_values, dim=-1) + local_values = global_values[..., vocab_start : vocab_start + local_vocab] + for reduce_loss in (False, True): + for overwrite_input in (False, True): + results = [] + for return_argmax in (False, True): + local_logits = local_values.clone().requires_grad_() + out = parallel_cross_entropy( + local_logits, + target, + label_smoothing=label_smoothing, + reduce_loss=reduce_loss, + dist_process_group=dist.group.WORLD, + overwrite_input=overwrite_input, + return_argmax=return_argmax, + ) + loss, argmax = out if return_argmax else (out, None) + loss.backward(torch.full_like(loss, 0.37)) + results.append((loss, local_logits.grad, argmax)) + (loss, grad, _), (argmax_loss, argmax_grad, argmax) = results + torch.testing.assert_close(argmax, expected_argmax, rtol=0, atol=0) + torch.testing.assert_close( + argmax_loss, loss, rtol=0, atol=0, equal_nan=True + ) + torch.testing.assert_close( + argmax_grad, grad, rtol=0, atol=0, equal_nan=True + ) + finally: + dist.destroy_process_group() + + @pytest.mark.parametrize("label_smoothing", [0.0, 0.1], ids=["plain", "smoothed"]) def test_parallel_cross_entropy_tensor_parallel(label_smoothing): """Validate tensor-parallel loss and gradients on two ranks.""" @@ -103,3 +167,20 @@ def test_parallel_cross_entropy_tensor_parallel(label_smoothing): nprocs=world_size, join=True, ) + + +@pytest.mark.parametrize("label_smoothing", [0.0, 0.1], ids=["plain", "smoothed"]) +def test_parallel_cross_entropy_tensor_parallel_argmax(label_smoothing): + """Validate the global argmax and the unchanged loss and gradients on two ranks.""" + + if torch.cuda.device_count() < 2: + pytest.skip("tensor-parallel cross entropy test requires two CUDA devices") + world_size = 2 + with tempfile.TemporaryDirectory() as temp_dir: + init_file = os.path.join(temp_dir, "distributed_init") + mp.spawn( + _run_tensor_parallel_argmax, + args=(world_size, init_file, label_smoothing), + nprocs=world_size, + join=True, + ) diff --git a/tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py b/tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py new file mode 100644 index 00000000000..5200ec1074b --- /dev/null +++ b/tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py @@ -0,0 +1,132 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""RMSNorm forward with MXFP8 output from Transformer Engine's fused kernel.""" + +import os + +import pytest +import torch + +import transformer_engine.pytorch as te +import transformer_engine_torch as tex +from transformer_engine.pytorch import MXFP8Quantizer +from transformer_engine.pytorch.constants import TE_DType + +from mxfp8_utils import swizzle_mxfp8_scale + +recipe_available, reason_for_no_recipe = te.is_mxfp8_available(return_reason=True) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.skipif( + os.getenv("NVTE_NORM_FWD_USE_CUDNN", "0") == "1", + reason="Tests Transformer Engine's fused kernel, not cuDNN", +) +@pytest.mark.parametrize("shape", [(128, 128), (256, 1024), (384, 7168)]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("fp8_dtype", [tex.DType.kFloat8E4M3, tex.DType.kFloat8E5M2]) +@pytest.mark.parametrize("zero_centered_gamma", [False, True]) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +def test_rmsnorm_fwd_mxfp8(shape, dtype, fp8_dtype, zero_centered_gamma, optimize_for_gemm): + """The fused output is the MXFP8 quantization of the FP32 normalized values.""" + torch.manual_seed(1234) + rows, cols = shape + eps = 1e-5 + x = torch.randn(rows, cols, dtype=dtype, device="cuda") + weight = torch.randn(cols, dtype=dtype, device="cuda") + + quantizer = MXFP8Quantizer(fp8_dtype) + quantizer.optimize_for_gemm = optimize_for_gemm + out, _, rsigma = tex.rmsnorm_fwd( + x, weight, eps, None, quantizer, TE_DType[dtype], 0, zero_centered_gamma + ) + assert out._with_gemm_swizzled_scales == optimize_for_gemm + + torch.testing.assert_close( + rsigma, torch.rsqrt(x.float().square().mean(dim=1) + eps), atol=0, rtol=1e-5 + ) + + # Reference: the MXFP8 quantizer on the FP32 normalized values, with the kernel's rsigma. + gamma = weight.float() + 1 if zero_centered_gamma else weight.float() + y = (x.float() * rsigma.unsqueeze(1)) * gamma + ref_quantizer = MXFP8Quantizer(fp8_dtype) + ref_quantizer.optimize_for_gemm = False + ref = ref_quantizer(y) + + for attr in ("_rowwise_data", "_columnwise_data"): + torch.testing.assert_close( + getattr(out, attr).view(torch.uint8), + getattr(ref, attr).view(torch.uint8), + atol=0, + rtol=0, + msg=attr, + ) + for attr, columnwise in (("_rowwise_scale_inv", False), ("_columnwise_scale_inv", True)): + expected = getattr(ref, attr) + if optimize_for_gemm: + expected = swizzle_mxfp8_scale(rows, cols, expected, columnwise=columnwise) + torch.testing.assert_close(getattr(out, attr), expected, atol=0, rtol=0, msg=attr) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.skipif( + os.getenv("NVTE_NORM_FWD_USE_CUDNN", "0") == "1", + reason="Tests Transformer Engine's fused kernel, not cuDNN", +) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +def test_rmsnorm_fwd_mxfp8_sm_margin(optimize_for_gemm): + """With all SMs but one reserved, the kernels run over chunks of rows with the same output.""" + torch.manual_seed(1234) + x = torch.randn(1024, 7168, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(7168, dtype=torch.bfloat16, device="cuda") + sm_count = torch.cuda.get_device_properties(x.device).multi_processor_count + + outputs = [] + for sm_margin in (0, sm_count - 1): + quantizer = MXFP8Quantizer(tex.DType.kFloat8E4M3) + quantizer.optimize_for_gemm = optimize_for_gemm + outputs.append( + tex.rmsnorm_fwd(x, weight, 1e-5, None, quantizer, TE_DType[x.dtype], sm_margin, False) + ) + (out, _, rsigma), (out_margin, _, rsigma_margin) = outputs + torch.testing.assert_close(rsigma_margin, rsigma, atol=0, rtol=0) + for attr in ( + "_rowwise_data", + "_columnwise_data", + "_rowwise_scale_inv", + "_columnwise_scale_inv", + ): + torch.testing.assert_close( + getattr(out_margin, attr), getattr(out, attr), atol=0, rtol=0, msg=attr + ) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_rmsnorm_fwd_mxfp8_2d_quantization(dtype): + """The fused kernels quantize 1D blocks, so with 2D quantization the forward normalizes and + then quantizes 32x32 blocks.""" + torch.manual_seed(1234) + x = torch.randn(256, 1024, dtype=dtype, device="cuda") + weight = torch.randn(1024, dtype=dtype, device="cuda") + + quantizer = MXFP8Quantizer(tex.DType.kFloat8E4M3, with_2d_quantization=True) + out, _, _ = tex.rmsnorm_fwd(x, weight, 1e-5, None, quantizer, TE_DType[dtype], 0, False) + + y, _, _ = tex.rmsnorm_fwd(x, weight, 1e-5, None, None, TE_DType[dtype], 0, False) + ref = MXFP8Quantizer(tex.DType.kFloat8E4M3, with_2d_quantization=True)(y) + for attr in ( + "_rowwise_data", + "_columnwise_data", + "_rowwise_scale_inv", + "_columnwise_scale_inv", + ): + torch.testing.assert_close( + getattr(out, attr).view(torch.uint8), + getattr(ref, attr).view(torch.uint8), + atol=0, + rtol=0, + msg=attr, + ) diff --git a/tests/pytorch/test_backward_override.py b/tests/pytorch/test_backward_override.py index c0acf2e6b3b..b0c56773f7b 100644 --- a/tests/pytorch/test_backward_override.py +++ b/tests/pytorch/test_backward_override.py @@ -813,21 +813,22 @@ def _run_grouped_linear_single_step_with_ctx_state( raise RuntimeError( "Output tensor has no grad_fn; cannot inspect grouped backward state." ) + state_holder = y.grad_fn.backward_objects required_attrs = ( "backward_override", "fp8", "reduce_and_update_bwd_fp8_tensors", ) - missing_attrs = [attr for attr in required_attrs if not hasattr(y.grad_fn, attr)] + missing_attrs = [attr for attr in required_attrs if not hasattr(state_holder, attr)] if missing_attrs: raise RuntimeError( "Grouped grad_fn does not expose required backward context attributes: " f"{', '.join(missing_attrs)}." ) ctx_state = ( - getattr(y.grad_fn, "backward_override"), - bool(getattr(y.grad_fn, "fp8")), - bool(getattr(y.grad_fn, "reduce_and_update_bwd_fp8_tensors")), + getattr(state_holder, "backward_override"), + bool(getattr(state_holder, "fp8")), + bool(getattr(state_holder, "reduce_and_update_bwd_fp8_tensors")), ) y.backward(dy) assert x_run.grad is not None diff --git a/tests/pytorch/test_cross_entropy.py b/tests/pytorch/test_cross_entropy.py index 0b2d21bbbbe..f34e6666d4b 100644 --- a/tests/pytorch/test_cross_entropy.py +++ b/tests/pytorch/test_cross_entropy.py @@ -430,3 +430,96 @@ def test_parallel_cross_entropy_deprecated_input_alias(): alias_loss = parallel_cross_entropy(logits, target, _input=logits) direct_loss = parallel_cross_entropy(logits, target) torch.testing.assert_close(alias_loss, direct_loss) + + +def _logits_with_argmax_cases(shape, dtype): + """BF16-representable logits whose rows exercise ties within and across 32K-wide blocks.""" + + torch.manual_seed(7) + values = torch.randn(shape, dtype=torch.bfloat16, device="cuda").to(dtype) + vocab = shape[-1] + peak = 8.0 + values[0, 0, [5, vocab - 30000]] = peak # equal maxima in different blocks + values[0, 1, [vocab - 30000, vocab - 4]] = peak # equal maxima in later blocks + values[0, 2, [100, 7]] = peak # equal maxima in the same block + values[0, 3, vocab - 1] = peak # a strictly larger maximum in the last block + values[0, 4] = 0.5 # a constant row + return values + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16], ids=["fp32", "bf16"]) +@pytest.mark.parametrize("reduce_loss", [False, True], ids=["none", "mean"]) +@pytest.mark.parametrize("label_smoothing", [0.0, 0.1], ids=["plain", "smoothed"]) +@pytest.mark.parametrize("overwrite_input", [False, True], ids=["safe", "destructive"]) +def test_parallel_cross_entropy_return_argmax(dtype, reduce_loss, label_smoothing, overwrite_input): + """The argmax matches torch.argmax, and loss and gradient do not change with it.""" + + shape = (2, 5, 70000) + values = _logits_with_argmax_cases(shape, dtype) + target = torch.randint(0, shape[-1], shape[:-1], device="cuda") + target[1, 1] = -100 + expected_argmax = torch.argmax(values, dim=-1) + + logits = values.clone().requires_grad_() + loss = parallel_cross_entropy( + logits, target, label_smoothing, reduce_loss, overwrite_input=overwrite_input + ) + argmax_logits = values.clone().requires_grad_() + argmax_loss, argmax = parallel_cross_entropy( + argmax_logits, + target, + label_smoothing, + reduce_loss, + overwrite_input=overwrite_input, + return_argmax=True, + ) + + assert argmax.dtype == torch.int64 + assert not argmax.requires_grad + torch.testing.assert_close(argmax, expected_argmax, rtol=0, atol=0) + + external_grad = torch.randn_like(loss) + loss.backward(external_grad) + argmax_loss.backward(external_grad) + torch.testing.assert_close(argmax_loss, loss, rtol=0, atol=0) + torch.testing.assert_close(argmax_logits.grad, logits.grad, rtol=0, atol=0) + + +def test_parallel_cross_entropy_return_argmax_transposed_input(): + """Each argmax belongs to the row of the loss next to it.""" + + values = _logits_with_argmax_cases((2, 5, 70000), torch.float32).transpose(0, 1) + target = torch.randint(0, values.shape[-1], values.shape[:-1], device="cuda") + loss, argmax = parallel_cross_entropy(values, target, return_argmax=True) + assert argmax.shape == loss.shape == values.shape[:-1] + torch.testing.assert_close(argmax, torch.argmax(values, dim=-1), rtol=0, atol=0) + + +def test_parallel_cross_entropy_return_argmax_nan(): + """As with torch.argmax, a NaN logit counts as the maximum and the first NaN wins, within a + block and across blocks.""" + + nan = float("nan") + small = torch.tensor( + [ + [8.0, -10.0, nan, -10.0], + [nan, 1.0, 5.0, 2.0], + [1.0, 5.0, 2.0, nan], + [3.0, nan, 3.0, nan], + [nan, nan, nan, nan], + [float("inf"), 1.0, nan, 2.0], + ], + device="cuda", + ) + large = torch.randn(4, 70000, device="cuda") + pair = torch.tensor([nan, 50.0], device="cuda") + large[0, [40000, 10]] = pair # NaN in a later block than the maximum + large[1, [10, 40000]] = pair # NaN in an earlier block than the maximum + large[2, [40001, 40000]] = pair # NaN after the maximum in the same block + large[3, [65000, 35000]] = nan # NaN in two blocks + for values in (small, large): + for dtype in (torch.float32, torch.bfloat16): + logits = values.to(dtype).unsqueeze(0) + target = torch.zeros(logits.shape[:-1], dtype=torch.long, device="cuda") + _, argmax = parallel_cross_entropy(logits, target, return_argmax=True) + torch.testing.assert_close(argmax, torch.argmax(logits, dim=-1), rtol=0, atol=0) diff --git a/tests/pytorch/test_fused_router.py b/tests/pytorch/test_fused_router.py index bd89ecebc9e..c6c82fa3d78 100644 --- a/tests/pytorch/test_fused_router.py +++ b/tests/pytorch/test_fused_router.py @@ -23,6 +23,70 @@ torch.cuda.manual_seed(seed) +@pytest.mark.parametrize("topk", [1, 10, 32]) +@pytest.mark.parametrize("histogram_mode", [None, "two_kernel", "fused_atomic"]) +@pytest.mark.parametrize("layout", ["bytemap", "bitmap_u8", "dense"]) +@pytest.mark.parametrize("mixed_scores", [False, True]) +def test_sigmoid_bias_preserves_tiny_scores(topk, histogram_mode, layout, mixed_scores): + """Bias affects selection, but must not round away the selected raw scores.""" + num_tokens, num_experts, num_bins = 17, 512, 1000 + logits = torch.linspace(-8.0, -4.0, num_experts, device="cuda").repeat(num_tokens, 1) + logits[:, :topk] = -16.0 - torch.arange(topk, device="cuda") % 9 + if mixed_scores: + logits[:, 0] = -1.0 + logits.requires_grad_(True) + expert_bias = torch.zeros(num_experts, device="cuda") + expert_bias[:topk] = 0.75 + 0.001 * torch.arange(topk, device="cuda") + expert_bias -= expert_bias.mean() + bin_bounds = torch.tensor([-1.0, 1.0], device="cuda") + histogram = torch.zeros(num_experts, num_bins, dtype=torch.int32, device="cuda") + reference_probs, reference_map = topk_score_function_pytorch( + logits, topk, scaling_factor=3.16, score_function="sigmoid", expert_bias=expert_bias + ) + kwargs = {} + if histogram_mode is not None: + kwargs.update( + qb_histogram=histogram, + qb_bin_bounds=bin_bounds, + qb_histogram_mode=histogram_mode, + ) + if layout == "dense": + kwargs["topk_indices"] = torch.empty(num_tokens, topk, dtype=torch.int16, device="cuda") + fused_logits = logits.detach().clone().requires_grad_(True) + probs, routing = fused_topk_with_score_function( + fused_logits, + topk, + False, + None, + None, + 3.16, + "sigmoid", + expert_bias, + routing_map_format="bitmap_u8" if layout == "bitmap_u8" else "bytemap", + **kwargs, + ) + if layout == "dense": + routing = topk_indices_to_routing_map(routing, num_experts) + elif layout == "bitmap_u8": + reference_map = _bytemap_to_bitmap_u8(reference_map) + torch.testing.assert_close(routing, reference_map) + # An absolute tolerance would hide precisely the tiny values under test. + torch.testing.assert_close(probs, reference_probs, atol=0, rtol=5e-6) + assert torch.all(probs[:, :topk] > 0) + if histogram_mode is not None: + reference = qb_topk_score_function_pytorch(logits, topk, expert_bias, bin_bounds, num_bins) + torch.testing.assert_close(histogram, reference["histogram"]) + # Nonuniform upstream gradients exercise the normalization derivative. + upstream = torch.linspace(-0.7, 1.3, num_experts, device="cuda").expand_as(probs) + probs.backward(upstream) + reference_probs.backward(upstream) + torch.testing.assert_close(fused_logits.grad, logits.grad, atol=2e-7, rtol=2e-4) + if mixed_scores and topk > 1: + torch.testing.assert_close( + fused_logits.grad[:, 1:topk], logits.grad[:, 1:topk], atol=0, rtol=2e-4 + ) + + def _get_tolerances(dtype: torch.dtype, num_experts: int): """Return (atol, rtol) scaled by the number of experts. diff --git a/tests/pytorch/test_fusible_ops.py b/tests/pytorch/test_fusible_ops.py index 34049523501..bdee6f7ffe8 100644 --- a/tests/pytorch/test_fusible_ops.py +++ b/tests/pytorch/test_fusible_ops.py @@ -48,6 +48,7 @@ # Import utility functions from utils import ( + MegatronTrainingHelper, assert_close, assert_close_grads, dtype_tols, @@ -1722,6 +1723,10 @@ def _test_basic_linear( op.weight.copy_(w_test) del w_test op.weight.main_grad = torch.full_like(op.weight, 0.5, dtype=torch.float32) + if accumulate_into_main_grad: + MegatronTrainingHelper.init_main_grad_buffers( + [op.weight], fill_value=0.5, overwrite_main_grad=False, zero_out_wgrad=True + ) with te.autocast(enabled=quantized_compute, recipe=recipe): y_test = forward(x_test) y_test.backward(dy_test) @@ -1739,13 +1744,9 @@ def _test_basic_linear( torch.testing.assert_close(y_test, y_ref, **tols) torch.testing.assert_close(dx_test, x_ref.grad, **tols) if accumulate_into_main_grad: - if op.weight.grad is not None: - torch.testing.assert_close( - op.weight.grad, - torch.zeros_like(op.weight.grad), - rtol=0, - atol=0, - ) + MegatronTrainingHelper.verify_main_grad_accumulation( + [op.weight], expected_main_grads=[w_ref.grad + 0.5], **tols + ) dw_test = op.weight.main_grad.to(dtype=torch.float64, device="cpu") - 0.5 else: dw_test = op.weight.grad.to(dtype=torch.float64, device="cpu") diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index f1ca21a4b2f..4ae63b44dbf 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -5,7 +5,6 @@ from __future__ import annotations from collections.abc import Iterable -import contextlib import functools import os import math @@ -2055,16 +2054,20 @@ def test_grouped_mlp_fp16( activation=activation, ) + @pytest.mark.parametrize( + "quantization", [q for q in _grouped_mlp_quantization_list if q is not None] + ) @pytest.mark.parametrize("bias", (False, True)) @pytest.mark.parametrize("runtime_offsets_supported", (False, True)) - def test_grouped_mlp_single_group_mxfp8( + def test_grouped_mlp_single_group( self, monkeypatch, *, + quantization: str, bias: bool, runtime_offsets_supported: bool, ) -> None: - """Single-group GroupedLinear + ScaledSwiGLU + GroupedLinear with MXFP8.""" + """Single-group GroupedLinear + ScaledSwiGLU + GroupedLinear.""" if ( runtime_offsets_supported and not grouped_mlp_module._cudnn_frontend_supports_single_group_runtime_offsets( @@ -2081,11 +2084,136 @@ def test_grouped_mlp_single_group_mxfp8( group_size=1, bias=bias, hidden_size=128, - quantization="mxfp8", + quantization=quantization, single_grouped_weight=False, activation="scaled_swiglu", ) + @pytest.mark.parametrize("mode", ("frozen_weights", "no_grad")) + @pytest.mark.parametrize("quantization", ("mxfp8", "nvfp4_rht")) + def test_grouped_mlp_single_group_no_weight_grad( + self, + *, + mode: str, + quantization: str, + hidden_size: int = 128, + tokens: int = 256, + dtype: torch.dtype = torch.bfloat16, + device: torch.device = "cuda", + ) -> None: + """Single-group fused grouped MLP when weight gradients are not needed + + Without weight gradients the FC2 input is quantized without + column-wise data, which must still run as a dense single-group + GEMM rather than a grouped cuBLAS GEMM. + """ + + maybe_skip_quantization( + quantization, dims=(tokens, hidden_size), device=device, dtype=dtype + ) + if not te.ops.fused.GroupedMLP_CuTeGEMMGLU.is_supported(): + pytest.skip("CuTe DSL fused grouped MLP is not supported") + + split_sizes = torch.tensor([tokens], dtype=torch.int, device=device) + in_shape = (tokens, hidden_size) + fc1_out_features = 2 * hidden_size + + x_ref, x_test = make_reference_and_test_tensors( + in_shape, + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="input"), + requires_grad=(mode == "frozen_weights"), + ) + dy_ref, dy_test = make_reference_and_test_tensors( + in_shape, + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="grad_output"), + requires_grad=False, + ) + probs_ref, probs_test = make_reference_and_test_tensors( + (tokens,), + test_dtype=dtype, + test_device=device, + requires_grad=False, + ) + fc1_w_ref, fc1_w_test = make_reference_and_test_tensors( + (fc1_out_features, hidden_size), + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="weight"), + requires_grad=False, + ) + fc2_w_ref, fc2_w_test = make_reference_and_test_tensors( + (hidden_size, hidden_size), + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="weight"), + requires_grad=False, + ) + + # Reference implementation + fc1_out = torch.nn.functional.linear(x_ref, fc1_w_ref) + fc1_out = fc1_out.reshape(tokens, fc1_out_features // 64, 2, 32).transpose(1, 2) + act_in1, act_in2 = fc1_out.reshape(tokens, fc1_out_features).chunk(2, dim=-1) + act_out = torch.nn.functional.silu(act_in1) * act_in2 + y_ref = torch.nn.functional.linear(act_out * probs_ref.unsqueeze(-1), fc2_w_ref) + if mode == "frozen_weights": + y_ref.backward(dy_ref) + + # Construct operations + recipe = make_recipe(quantization) + with te.quantized_model_init(enabled=True, recipe=recipe): + fc1 = te.ops.GroupedLinear( + 1, hidden_size, fc1_out_features, bias=False, device=device, dtype=dtype + ) + fc2 = te.ops.GroupedLinear( + 1, hidden_size, hidden_size, bias=False, device=device, dtype=dtype + ) + module = te.ops.Sequential(fc1, te.ops.ScaledSwiGLU(glu_interleave_size=32), fc2) + with torch.no_grad(): + fc1.weight0.copy_(fc1_w_test) + fc2.weight0.copy_(fc2_w_test) + for param in module.parameters(): + param.requires_grad_(False) + del fc1_w_test, fc2_w_test + + # Forward (and backward w.r.t. input only for frozen weights) + with te.autocast(enabled=True, recipe=recipe): + if mode == "no_grad": + with torch.no_grad(): + y_test = module(x_test, split_sizes, probs_test, split_sizes) + else: + y_test = module(x_test, split_sizes, probs_test, split_sizes) + if mode == "frozen_weights": + y_test.backward(dy_test) + + # Check that operations were fused + forward_ops = module._module_groups[0]._forward_ops + assert len(forward_ops) == 1 + assert isinstance(forward_ops[0][0], te.ops.fused.GroupedMLP_CuTeGEMMGLU) + + # Loose tols for sanity checking + tols = {"rtol": 0.125, "atol": 0.25} + assert_close(y_test, y_ref, **tols) + if mode == "frozen_weights": + assert_close_grads(x_test, x_ref, **tols) + assert fc1.weight0.grad is None + assert fc2.weight0.grad is None + @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) def test_single_grouped_weight_eval_preserves_columnwise_usage( self, @@ -3097,8 +3225,9 @@ def train_step( class TestGroupedMLPDeterminism: """Determinism coverage for the CuTe DSL fused grouped MLP. - Only the dSReLU wrapper can make ``dprob`` bit-exact, and only from cuDNN FE 1.28.0 on. - Anything else must refuse a determinism request rather than run non-deterministically. + The dSReLU and dGLU wrappers can make ``dprob`` bit-exact when the installed cuDNN FE + accepts ``deterministic``. Anything else must refuse a determinism request rather than run + non-deterministically. """ @pytest.fixture @@ -3136,13 +3265,6 @@ def test_either_knob_requests_determinism( torch.use_deterministic_algorithms(torch_flag) assert grouped_mlp_module._deterministic_algorithms_required() is expected - def test_only_the_srelu_path_can_be_deterministic(self) -> None: - """The capability belongs to the wrapper, not the environment. Needs no GPU.""" - glu = grouped_mlp_module.GroupedMLP_CuTeGEMMGLU - unary = grouped_mlp_module.GroupedMLP_CuTeGEMMUnary - assert glu.grouped_gemm_dactivation_is_deterministic() is False - assert isinstance(unary.grouped_gemm_dactivation_is_deterministic(), bool) - @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) @pytest.mark.parametrize( "activation,fused_cls", @@ -3159,12 +3281,7 @@ def test_determinism_either_runs_or_refuses( pytest.skip("MXFP8 fused grouped MLP is not supported on this system") monkeypatch.setenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") - expectation = ( - contextlib.nullcontext() - if fused_cls.grouped_gemm_dactivation_is_deterministic() - else pytest.raises(RuntimeError, match="dprob") - ) - with expectation: + try: TestGroupedMLPFusedOp().test_grouped_mlp( bias=False, hidden_size=128, @@ -3172,6 +3289,14 @@ def test_determinism_either_runs_or_refuses( single_grouped_weight=False, activation=activation, ) + except RuntimeError as e: + # Refused loudly: TE's own check, or cuDNN declining `deterministic=True` for this + # configuration (NotImplementedError, e.g. no Rubin kernel). Anything else is a bug. + assert isinstance(e, NotImplementedError) or ( + grouped_mlp_module._DETERMINISM_REQUESTED in str(e) + ) + return + assert fused_cls.grouped_gemm_dactivation_is_deterministic() @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) def test_scale_bias_refuses_under_the_torch_flag( @@ -3199,17 +3324,23 @@ def test_scale_bias_refuses_under_the_torch_flag( ) @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) - def test_dprob_is_bit_exact_across_runs(self, monkeypatch) -> None: + @pytest.mark.parametrize( + "activation,fused_cls", + ( + ("scaled_srelu", grouped_mlp_module.GroupedMLP_CuTeGEMMUnary), + ("scaled_swiglu", grouped_mlp_module.GroupedMLP_CuTeGEMMGLU), + ), + ) + def test_dprob_is_bit_exact_across_runs(self, monkeypatch, *, activation, fused_cls) -> None: """Repeated identical runs must give a bit-identical ``dprob``. An ulp of reordering passes every tolerance in this file, so only an exact comparison across runs can see it. """ - fused_cls = grouped_mlp_module.GroupedMLP_CuTeGEMMUnary if not fused_cls.is_supported(): pytest.skip("MXFP8 fused grouped MLP is not supported on this system") if not fused_cls.grouped_gemm_dactivation_is_deterministic(): - pytest.skip("dSReLU determinism needs cuDNN frontend 1.28.0 or later") + pytest.skip("the installed cuDNN frontend cannot make this dactivation deterministic") monkeypatch.setenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") @@ -3240,14 +3371,29 @@ def _rand(*shape, requires_grad=True) -> torch.Tensor: probs = _rand(num_tokens) # No bias, or probs.grad comes from the Triton dbias kernel instead of cuDNN. + # GLU's FC1 produces both gate halves. GLU uses one grouped weight (dense), which + # cudnn-frontend#1319 already supports; discrete weights need cudnn-frontend#1414. + is_glu = fused_cls is grouped_mlp_module.GroupedMLP_CuTeGEMMGLU with te.quantized_model_init(enabled=True, recipe=recipe): module = te.ops.Sequential( te.ops.GroupedLinear( - group_size, hidden_size, hidden_size, bias=False, device=device, dtype=dtype + group_size, + hidden_size, + 2 * hidden_size if is_glu else hidden_size, + bias=False, + single_grouped_weight=is_glu, + device=device, + dtype=dtype, ), - te.ops.ScaledSReLU(), + _make_scaled_activation(activation, glu_interleave_size=32 if is_glu else None), te.ops.GroupedLinear( - group_size, hidden_size, hidden_size, bias=False, device=device, dtype=dtype + group_size, + hidden_size, + hidden_size, + bias=False, + single_grouped_weight=is_glu, + device=device, + dtype=dtype, ), ) diff --git a/tests/pytorch/test_grouped_tensor.py b/tests/pytorch/test_grouped_tensor.py index 388b3bb7041..ae4a60eb270 100644 --- a/tests/pytorch/test_grouped_tensor.py +++ b/tests/pytorch/test_grouped_tensor.py @@ -629,11 +629,15 @@ def test_quantize_varying_shapes(self, quantization: str) -> None: "shape_case", ["varying_first", "varying_last", "varying_both"], ) + @pytest.mark.parametrize("usage", ["rowwise", "columnwise", "both"]) + @pytest.mark.parametrize("optimize_for_gemm", [False, True]) @pytest.mark.parametrize("output_dbias", [False, True]) @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) - def test_quantize_grouped_mxfp8(self, shape_case: str, output_dbias: bool) -> None: + def test_quantize_grouped_mxfp8( + self, shape_case: str, usage: str, optimize_for_gemm: bool, output_dbias: bool + ) -> None: """Test grouped MXFP8 quantization against per-tensor quantization for - varying first/last/both dimensions.""" + varying first/last/both dimensions and compact or GEMM-swizzled scales.""" if output_dbias and shape_case != "varying_first": pytest.skip("bgrad_group_quantize requires constant last dimension") @@ -688,7 +692,10 @@ def test_quantize_grouped_mxfp8(self, shape_case: str, output_dbias: bool) -> No ) quantizer = MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3) - quantizer.set_usage(rowwise=True, columnwise=False) + rowwise = usage in ("rowwise", "both") + columnwise = usage in ("columnwise", "both") + quantizer.set_usage(rowwise=rowwise, columnwise=columnwise) + quantizer.optimize_for_gemm = optimize_for_gemm if output_dbias: grouped_output, dbias = tex.bgrad_group_quantize( @@ -700,19 +707,23 @@ def test_quantize_grouped_mxfp8(self, shape_case: str, output_dbias: bool) -> No ) # Reference: quantize each tensor independently and concatenate the - # rowwise data / scale_inv buffers in tensor order. - expected_data = [] - expected_scale_inv = [] - for tensor in input_tensors: - qtensor = quantizer(tensor) - expected_data.append(qtensor._rowwise_data.reshape(-1)) - expected_scale_inv.append(qtensor._rowwise_scale_inv.reshape(-1)) - - expected_data = torch.cat(expected_data) - expected_scale_inv = torch.cat(expected_scale_inv) - - assert torch.equal(grouped_output.rowwise_data, expected_data) - assert torch.equal(grouped_output.scale_inv, expected_scale_inv) + # data / scale_inv buffers in tensor order. Both dimensions of every member + # are multiples of 128, so swizzled scales need no padding between members. + expected_tensors = [quantizer(tensor) for tensor in input_tensors] + if rowwise: + expected_data = torch.cat([t._rowwise_data.reshape(-1) for t in expected_tensors]) + expected_scales = torch.cat( + [t._rowwise_scale_inv.reshape(-1) for t in expected_tensors] + ) + assert torch.equal(grouped_output.rowwise_data, expected_data) + assert torch.equal(grouped_output.scale_inv, expected_scales) + if columnwise: + expected_data = torch.cat([t._columnwise_data.reshape(-1) for t in expected_tensors]) + expected_scales = torch.cat( + [t._columnwise_scale_inv.reshape(-1) for t in expected_tensors] + ) + assert torch.equal(grouped_output.columnwise_data, expected_data) + assert torch.equal(grouped_output.columnwise_scale_inv, expected_scales) if output_dbias: expected_dbias = torch.stack([t.sum(dim=0) for t in input_tensors]) diff --git a/tests/pytorch/test_recipe.py b/tests/pytorch/test_recipe.py index 40983c89370..446bfda8e59 100644 --- a/tests/pytorch/test_recipe.py +++ b/tests/pytorch/test_recipe.py @@ -782,3 +782,41 @@ def test_stateful_unknown_or_malformed_pickled_extra_state_requires_opt_in(paylo monkeypatch.setenv(UNSAFE_PICKLE_EXTRA_STATE_ENV, "1") assert should_load_extra_state_pickle(payload, "test") + + +@pytest.mark.parametrize("use_power_2_scales", [False, True]) +def test_float8_current_scaling_quant_params_follow_constructor(use_power_2_scales): + recipe = Float8CurrentScaling(use_power_2_scales=use_power_2_scales) + for qparams in ( + recipe.fp8_quant_fwd_inp, + recipe.fp8_quant_fwd_weight, + recipe.fp8_quant_bwd_grad, + ): + assert qparams.power_2_scale == use_power_2_scales + + +@pytest.mark.parametrize("use_f32_scales", [False, True]) +def test_float8_block_scaling_quant_params_follow_constructor(use_f32_scales): + recipe = Float8BlockScaling(use_f32_scales=use_f32_scales) + for qparams in ( + recipe.fp8_quant_fwd_inp, + recipe.fp8_quant_fwd_weight, + recipe.fp8_quant_bwd_grad, + ): + assert qparams.power_2_scale == (not use_f32_scales) + + +@pytest.mark.parametrize( + "recipe", + [Float8CurrentScaling(use_power_2_scales=True), Float8BlockScaling(use_f32_scales=True)], + ids=["current_scaling", "block_scaling"], +) +def test_fp8_recipe_unpickle_rebuilds_missing_quant_params(recipe): + state = dict(recipe.__dict__) + for name in ("fp8_quant_fwd_inp", "fp8_quant_fwd_weight", "fp8_quant_bwd_grad"): + del state[name] + restored = recipe.__class__.__new__(recipe.__class__) + restored.__setstate__(state) + assert restored.fp8_quant_fwd_inp.power_2_scale == recipe.fp8_quant_fwd_inp.power_2_scale + assert restored.fp8_quant_fwd_weight.power_2_scale == recipe.fp8_quant_fwd_weight.power_2_scale + assert restored.fp8_quant_bwd_grad.power_2_scale == recipe.fp8_quant_bwd_grad.power_2_scale diff --git a/tests/pytorch/test_sanity.py b/tests/pytorch/test_sanity.py index 7db3002427c..8022623d0ae 100644 --- a/tests/pytorch/test_sanity.py +++ b/tests/pytorch/test_sanity.py @@ -1186,6 +1186,96 @@ def test_sanity_gradient_accumulation_fusion(dtype, fp8_recipe, model, skip_wgra _test_sanity_e2e_gradient_accumulation_fusion(block, dtype, config, fp8_recipe, skip_wgrad) +def _wgrad_fusion_module(module: str, hidden: int, dtype: torch.dtype): + """A module that accumulates its weight gradients into ``main_grad``, and its forward.""" + kwargs = dict(params_dtype=dtype, device="cuda", fuse_wgrad_accumulation=True) + if module == "Linear": + block = Linear(hidden, hidden, bias=False, **kwargs) + elif module == "LayerNormLinear": + block = LayerNormLinear(hidden, hidden, bias=False, **kwargs) + elif module == "LayerNormMLP": + block = LayerNormMLP(hidden, 2 * hidden, bias=False, **kwargs) + elif module == "GroupedLinear": + block = GroupedLinear(2, hidden, hidden, bias=False, **kwargs) + return block, lambda x: block(x, [x.shape[0] // 2, x.shape[0] - x.shape[0] // 2]) + else: + block = te.ops.Linear( + hidden, hidden, bias=False, device="cuda", dtype=dtype, accumulate_into_main_grad=True + ) + return block, block + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Needs two GPUs.") +# GroupedLinear is left out: its multi-stream cuBLAS GEMM keeps the streams of the first device +# it runs on, so its forward pass fails on a second device before any weight gradient. +@pytest.mark.parametrize("module", ["Linear", "LayerNormLinear", "LayerNormMLP", "ops.Linear"]) +@pytest.mark.parametrize("zero_out_wgrad", all_boolean) +def test_sanity_dummy_wgrad_device(module, zero_out_wgrad): + """The cached dummy weight gradient is on the weight's device when the same module shape + runs backward on one GPU and then on another.""" + dtype, hidden, tokens = torch.bfloat16, 128, 64 + for device in (0, 1): + with torch.cuda.device(device): + block, forward = _wgrad_fusion_module(module, hidden, dtype) + weights = [ + p + for name, p in block.named_parameters() + if "weight" in name and "layer_norm" not in name + ] + for p in weights: + p.main_grad = torch.zeros(p.shape, dtype=torch.float32, device="cuda") + p.grad_added_to_main_grad = False + p.zero_out_wgrad = zero_out_wgrad + x = torch.randn(tokens, hidden, dtype=dtype, device="cuda") + forward(x).backward(torch.randn(tokens, hidden, dtype=dtype, device="cuda")) + for p in weights: + assert p.grad is not None and p.grad.device == p.device + + +@pytest.mark.parametrize( + "module", ["Linear", "LayerNormLinear", "LayerNormMLP", "GroupedLinear", "ops.Linear"] +) +@pytest.mark.parametrize("zero_out_wgrad", all_boolean) +def test_sanity_zero_tensor_dummy_wgrad(module, zero_out_wgrad): + """``zero_tensor_dummy_wgrad`` makes the dummy weight gradient a ZeroTensor. + + The module runs twice before one backward pass, as an LM head shared by the main and MTP + heads does, so autograd adds the two dummy weight gradients. ``main_grad`` must not + change, and ``zero_out_wgrad`` keeps its zero-filled dummy. + """ + dtype, hidden, tokens = torch.bfloat16, 128, 64 + torch.manual_seed(seed) + block, forward = _wgrad_fusion_module(module, hidden, dtype) + weights = [ + p for name, p in block.named_parameters() if "weight" in name and "layer_norm" not in name + ] + inputs = [torch.randn(tokens, hidden, dtype=dtype, device="cuda") for _ in range(2)] + grad_outputs = [torch.randn(tokens, hidden, dtype=dtype, device="cuda") for _ in range(2)] + + main_grads = {} + for zero_tensor_dummy_wgrad in (False, True): + for p in weights: + p.main_grad = torch.zeros(p.shape, dtype=torch.float32, device="cuda") + p.grad_added_to_main_grad = False + p.zero_out_wgrad = zero_out_wgrad + p.zero_tensor_dummy_wgrad = zero_tensor_dummy_wgrad + p.grad = None + outputs = [forward(x) for x in inputs] + torch.autograd.backward(outputs, grad_outputs) + main_grads[zero_tensor_dummy_wgrad] = [p.main_grad for p in weights] + for p in weights: + assert p.grad_added_to_main_grad + assert p.grad is not None + if zero_tensor_dummy_wgrad and not zero_out_wgrad: + assert p.grad._is_zerotensor() + elif zero_out_wgrad: + assert not p.grad._is_zerotensor() + assert torch.count_nonzero(p.grad) == 0 + for reference, main_grad in zip(main_grads[False], main_grads[True]): + assert torch.count_nonzero(reference) > 0 + torch.testing.assert_close(main_grad, reference, rtol=0, atol=0) + + def test_model_multiple_cast(): a = torch.zeros((16, 16), device="cuda") m = Linear(16, 32) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index eae6f0a8a24..8299c98803a 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -4,12 +4,15 @@ import abc import contextlib +import copy import dataclasses import os import re +import subprocess import sys +import textwrap import warnings -from typing import Union +from typing import Literal, NamedTuple, Union import pytest import torch @@ -697,6 +700,9 @@ def _packed_layout(qkv_format: str, packed_dim: int, interleave_dim: int) -> str _DPA_COMPILE_CONFIGS = { "self_bshd_causal": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="causal")), + "self_bshd_causal_dropout": _cfg( + ModelConfig(2, 128, 4, 64, attn_mask_type="causal", dropout_p=0.1) + ), "self_sbhd_no_mask": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="no_mask"), "sbhd"), "self_bshd_swa": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="causal", window_size=(16, 0))), "gqa_bshd_causal": _cfg(ModelConfig(2, 128, 8, 64, num_gqa_groups=2, attn_mask_type="causal")), @@ -717,6 +723,21 @@ def _packed_layout(qkv_format: str, packed_dim: int, interleave_dim: int) -> str ModelConfig(2, 128, 4, 64, attn_mask_type="causal"), packed="qkv", interleave_dim=-2 ), "packed_kv_bshd_bs2hd": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="causal"), packed="kv"), + "packed_kv_bshd_bsh2d": _cfg( + ModelConfig(2, 128, 4, 64, attn_mask_type="causal"), packed="kv", interleave_dim=-2 + ), + "packed_qkv_bs3hd_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="qkv" + ), + "packed_qkv_bsh3d_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="qkv", interleave_dim=-2 + ), + "packed_kv_bshd_bs2hd_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="kv" + ), + "packed_kv_bshd_bsh2d_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="kv", interleave_dim=-2 + ), "packed_kv_thd_th2d": _cfg( ModelConfig(2, 128, 4, 64, attn_mask_type="padding_causal"), "thd", @@ -901,16 +922,8 @@ def _packed(seqlen, tokens, packed_dim): return args, kwargs, grad_tensors -def _skip_unsupported( - spec: dict, backend: str, dtype, compiled: bool = True, inference_params=None -) -> None: - """Skip what the backend under test cannot run, or -- for a test that - compiles it -- cannot be compiled.""" - if compiled and backend == "fused": - # FusedAttention's forward carries @no_torch_dynamo, so there is nothing - # to compile: it runs as an eager island. Drop this skip once it traces, - # and the tests below cover it as they do the others. - pytest.skip("FusedAttention is an eager island and does not compile") +def _skip_unsupported(spec: dict, backend: str, dtype, inference_params=None) -> None: + """Skip configurations the backend under test cannot run.""" available, _, _ = get_available_attention_backends( spec["model_config"], dtype, @@ -1018,12 +1031,17 @@ def _compare_compiled_to_eager( ) -> None: """Run the module eagerly and compiled on the same inputs, and compare both the output and every input gradient.""" + cpu_rng_state = torch.get_rng_state() + cuda_rng_state = torch.cuda.get_rng_state() eager = _run_and_capture(module, args, kwargs, grads) torch._dynamo.reset() # Force backend selection to be re-run (and traced) inside the compiled # region instead of being served from the cache the eager call populated. _force_dpa_backend(monkeypatch, backend) + # Reuse the eager dropout mask when comparing outputs and gradients. + torch.set_rng_state(cpu_rng_state) + torch.cuda.set_rng_state(cuda_rng_state) compiled = _run_and_capture(torch.compile(module, **compile_kwargs), args, kwargs, grads) _assert_dpa_backend(backend) @@ -1032,7 +1050,8 @@ def _compare_compiled_to_eager( @pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) @pytest.mark.parametrize("config", _DPA_COMPILE_CONFIGS.keys()) -def test_dpa_torch_compile(monkeypatch, backend, config): +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_dpa_torch_compile(monkeypatch, backend, config, dtype): """`DotProductAttention` under `torch.compile(fullgraph=True)` must match eager in forward and backward, for every backend that supports the configuration. @@ -1041,45 +1060,59 @@ def test_dpa_torch_compile(monkeypatch, backend, config): whole module: input unpacking, qkv layout, backend selection and the backend itself. """ - dtype = torch.bfloat16 spec = _DPA_COMPILE_CONFIGS[config] module = _make_dpa(spec, dtype) args, kwargs, grads = _make_dpa_inputs(spec, dtype) _skip_unsupported(spec, backend, dtype, inference_params=kwargs.get("inference_params")) _force_dpa_backend(monkeypatch, backend) + # Inductor uses a different RNG for unfused dropout by default. Keep the + # eager RNG here so this numerical comparison uses the same dropout mask. + options = ( + {"fallback_random": True} if backend == "unfused" and spec["model_config"].dropout_p else {} + ) _compare_compiled_to_eager( - module, args, kwargs, grads, monkeypatch, backend, dtype, fullgraph=True + module, args, kwargs, grads, monkeypatch, backend, dtype, fullgraph=True, options=options ) -def test_dpa_torch_compile_around_fused(monkeypatch): - """FusedAttention itself is an eager island, but everything around it is - compiled: DotProductAttention traces up to the backend call, breaks the - graph there and resumes afterwards. What crosses that break has to survive - it -- the sub-backend enum did not, and reached cuDNN as the function that - produced it.""" +def test_dpa_torch_compile_fused_op_unavailable(monkeypatch): + """Without the fused attention custom op, FusedAttention is an eager island + and everything around it is compiled: DotProductAttention traces up to the + backend call, breaks the graph there and resumes afterwards. What crosses + that break has to survive it -- the sub-backend enum did not, and reached + cuDNN as the function that produced it.""" + from transformer_engine.pytorch.attention.dot_product_attention import backends + dtype = torch.bfloat16 spec = _DPA_COMPILE_CONFIGS["self_bshd_causal"] - _skip_unsupported(spec, "fused", dtype, compiled=False) + _skip_unsupported(spec, "fused", dtype) _force_dpa_backend(monkeypatch, "fused") + monkeypatch.setattr(backends, "_fused_attn_op", None) module = _make_dpa(spec, dtype) args, kwargs, grads = _make_dpa_inputs(spec, dtype) # No fullgraph: the graph break at the eager island is the point here. - _compare_compiled_to_eager(module, args, kwargs, grads, monkeypatch, "fused", dtype) + with pytest.warns(UserWarning, match="Falling back to eager execution"): + _compare_compiled_to_eager(module, args, kwargs, grads, monkeypatch, "fused", dtype) -@pytest.mark.parametrize("backend", ["flash", "unfused"]) -@pytest.mark.parametrize("config", ["self_bshd_causal", "kv_cache_bshd"]) +@pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) +@pytest.mark.parametrize( + "config", + ["self_bshd_causal", "kv_cache_bshd", "packed_qkv_bsh3d", "packed_kv_bshd_bsh2d"], +) def test_dpa_torch_compile_cudagraphs(monkeypatch, backend, config): """`mode="reduce-overhead"`: forward and backward of DotProductAttention are captured into CUDA graphs and replayed on subsequent iterations.""" dtype = torch.bfloat16 spec = _DPA_COMPILE_CONFIGS[config] - _force_dpa_backend(monkeypatch, backend) - module = _make_dpa(spec, dtype) + _, kwargs, _ = _make_dpa_inputs(spec, dtype) + # Before forcing the backend: probing the available backends re-runs the + # selection and would otherwise be cached over the forced one. + _skip_unsupported(spec, backend, dtype, inference_params=kwargs.get("inference_params")) + _force_dpa_backend(monkeypatch, backend) torch._dynamo.reset() counters.clear() @@ -1099,7 +1132,7 @@ def test_dpa_torch_compile_cudagraphs(monkeypatch, backend, config): assert not counters["inductor"]["cudagraph_skips"], "inductor skipped CUDA graphs" -@pytest.mark.parametrize("backend", ["flash", "unfused"]) +@pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) @pytest.mark.parametrize("paged", [False, True], ids=["non_paged", "paged"]) @pytest.mark.parametrize("cuda_graphs", [False, True], ids=["default", "cudagraphs"]) def test_dpa_torch_compile_kv_cache_decoding(monkeypatch, backend, paged, cuda_graphs): @@ -1238,7 +1271,7 @@ def _thd_without_max_seqlen_inputs(spec, dtype): } -@pytest.mark.parametrize("backend", ["flash", "unfused"]) +@pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) @pytest.mark.parametrize("case", _EAGER_FALLBACK_CASES.keys()) def test_dpa_torch_compile_eager_fallback(monkeypatch, backend, case): """Calls that cannot be traced run as an eager island instead, with a @@ -1249,6 +1282,7 @@ def test_dpa_torch_compile_eager_fallback(monkeypatch, backend, case): dtype = torch.bfloat16 config_name, make_inputs = _EAGER_FALLBACK_CASES[case] spec = _DPA_COMPILE_CONFIGS[config_name] + _skip_unsupported(spec, backend, dtype) _force_dpa_backend(monkeypatch, backend) module = _make_dpa(spec, dtype) @@ -2380,6 +2414,899 @@ def fn(inp): ) +# --------------------------------------------------------------------------- +# te.GroupedLinear +# --------------------------------------------------------------------------- + +_GROUPED_M_SPLITS = [32, 16, 48, 32] # FP8-legal splits (each divisible by 8) +_GROUPED_NUM_GEMMS = len(_GROUPED_M_SPLITS) +_GROUPED_IN, _GROUPED_OUT = 64, 32 + + +def _grouped_input(dtype, device, requires_grad=False): + return torch.randn( + sum(_GROUPED_M_SPLITS), + _GROUPED_IN, + dtype=dtype, + device=device, + requires_grad=requires_grad, + ) + + +def _assert_close_grouped(fn, compiled, model, base, ref_model=None): + """Run ``fn`` eagerly and ``compiled`` on identical inputs; assert the + forward output, the input gradient and every expert's weight / bias + gradients match.""" + ref_model = model if ref_model is None else ref_model + inp_eager = base.detach().clone().requires_grad_(True) + ref_model.zero_grad(set_to_none=True) + out_eager = fn(inp_eager) + grad_output = torch.randn_like(out_eager) + out_eager.backward(grad_output) + ref_out = out_eager.detach().clone() + ref_igrad = inp_eager.grad.detach().clone() + ref_grads = [param.grad.detach().clone() for param in ref_model.parameters()] + + inp_compiled = base.detach().clone().requires_grad_(True) + model.zero_grad(set_to_none=True) + # Clone before a later cuda-graph replay overwrites the static output buffer. + out_compiled = compiled(inp_compiled).clone() + out_compiled.backward(grad_output) + + torch.testing.assert_close(out_compiled, ref_out, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + torch.testing.assert_close(inp_compiled.grad, ref_igrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + for (name, param), ref_grad in zip(model.named_parameters(), ref_grads): + torch.testing.assert_close( + param.grad, + ref_grad, + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + msg=lambda msg: f"gradient mismatch for {name}: {msg}", + ) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize( + "fp8_recipe", + [None, *_all_recipes], + ids=lambda r: "bf16" if r is None else type(r).__name__, +) +def test_te_grouped_linear_compiles(fp8_recipe, compile_mode): + """ + torch.compile(fullgraph=True) of ``te.GroupedLinear`` under every built-in + recipe (plus the bf16-only baseline), for both the default backend and + ``mode="reduce-overhead"`` (CUDA-graph trees). + """ + dtype = torch.bfloat16 + device = "cuda" + if fp8_recipe is not None and fp8_recipe.nvfp4(): + fp8_recipe = dataclasses.replace(fp8_recipe, disable_stochastic_rounding=True) + block_scaled = fp8_recipe is not None and not fp8_recipe.float8_current_scaling() + m_splits = [128, 256, 128, 256] if block_scaled else _GROUPED_M_SPLITS + in_features, out_features = (128, 128) if block_scaled else (_GROUPED_IN, _GROUPED_OUT) + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, in_features, out_features, params_dtype=dtype, device=device + ) + + def make_input(): + return torch.randn( + sum(m_splits), in_features, dtype=dtype, device=device, requires_grad=True + ) + + def fn(inp): + if fp8_recipe is None: + return model(inp, m_splits) + with te.autocast(recipe=fp8_recipe): + return model(inp, m_splits) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, make_input(), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + n_iters = 3 if compile_mode == "reduce-overhead" else 1 + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for _ in range(n_iters): + _assert_close_grouped(fn, compiled, model, make_input()) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize("compile_mode", _compile_modes) +def test_te_grouped_linear_compile_with_quantized_fp8_weight(compile_mode): + """torch.compile of GroupedLinear with the weights initialized as FP8 + tensors (exercises the wrapper op's input flattening inside a ``Tensor[]`` + slot group).""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + + with te.quantized_model_init(enabled=True, recipe=fp8_recipe): + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, + _GROUPED_IN, + _GROUPED_OUT, + params_dtype=dtype, + device=device, + ) + assert isinstance(model.weight0, te.Float8Tensor) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + n_iters = 3 if compile_mode == "reduce-overhead" else 1 + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for _ in range(n_iters): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize("compile_mode", _compile_modes) +def test_te_grouped_linear_compile_is_first_microbatch(compile_mode): + """torch.compile of ``te.GroupedLinear`` across a microbatch schedule: + ``is_first_microbatch=True`` caches every expert's FP8 weight, later steps + must reuse the caches and stay numerically aligned with eager.""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + ref_model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + with torch.no_grad(): + for i in range(_GROUPED_NUM_GEMMS): + getattr(ref_model, f"weight{i}").copy_(getattr(model, f"weight{i}")) + getattr(ref_model, f"bias{i}").copy_(getattr(model, f"bias{i}")) + + schedule = [True, False, False] + is_first = schedule[0] # rebound each step; closed over by the fns. + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS, is_first_microbatch=is_first) + + def ref_fn(inp): + with te.autocast(recipe=fp8_recipe): + return ref_model(inp, _GROUPED_M_SPLITS, is_first_microbatch=is_first) + + # Eager priming: FP8 state must exist before tracing (creating quantizers + # in-graph breaks later recompiles; upstream Dynamo bug). + is_first = None + fn(_grouped_input(dtype, device, requires_grad=True)) + is_first = schedule[0] + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + cached_workspaces = None + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for step, is_first in enumerate(schedule): + base = _grouped_input(dtype, device) + + _assert_close_grouped(ref_fn, compiled, model, base, ref_model=ref_model) + + workspaces = [ + model._fp8_workspaces.get(f"weight{i}") for i in range(_GROUPED_NUM_GEMMS) + ] + assert all( + ws is not None for ws in workspaces + ), f"missing cached FP8 weight(s) after step {step}" + if step == 0: + cached_workspaces = workspaces + else: + for i in range(_GROUPED_NUM_GEMMS): + assert ( + workspaces[i] is cached_workspaces[i] + ), f"cache for expert {i} rebuilt at step {step}" + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +def test_te_grouped_linear_m_splits_change(): + """Unchanged ``m_splits`` reuse the graph. A changed distribution makes + Dynamo mark the values dynamic; they cross the op boundary as a + ``SymInt[]`` slot, so the (recompiled, now symbolic) graph still runs the + compiled path with eager numerics -- for any later distribution too.""" + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + m_splits = list(_GROUPED_M_SPLITS) + + def fn(inp): + return model(inp, m_splits) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + + # Two warmup calls: the second absorbs the one-time recompile from module + # attributes lazily created during call one. + for _ in range(2): + compiled(_grouped_input(dtype, device, requires_grad=True)).sum().backward() + model.zero_grad(set_to_none=True) + baseline = _dynamo_counter("stats", "unique_graphs") + + # Same splits: no recompile. + compiled(_grouped_input(dtype, device, requires_grad=True)).sum().backward() + model.zero_grad(set_to_none=True) + if baseline: + assert _dynamo_counter("stats", "unique_graphs") == baseline + + # New split distributions (same total): compiled path, same numerics. + m_splits[:2] = [16, 32] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + m_splits[2:] = [40, 40] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + m_splits[:] = [24, 40, 32, 32] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + baseline = _dynamo_counter("stats", "unique_graphs") + m_splits[:] = [40, 24, 48, 16] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + if baseline: + assert _dynamo_counter("stats", "unique_graphs") == baseline + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_te_grouped_linear_compile_train_eval_switch(): + """train -> eval -> train on the same compiled ``te.GroupedLinear``, vs eager.""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS, is_first_microbatch=True) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + + def train_step(): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + model.zero_grad(set_to_none=True) + + train_step() + + model.eval() + x = _grouped_input(dtype, device) + with torch.no_grad(): + out_eval = compiled(x) + ref_eval = fn(x) + torch.testing.assert_close(out_eval, ref_eval, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + + model.train() + train_step() + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +def test_te_grouped_linear_compile_m_splits_tensor_falls_back(): + """A device-tensor ``m_splits`` inside the compiled region cannot cross the + op boundary (its values would be unbacked); the module must warn and fall + back to eager (a graph break).""" + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + m_splits_tensor = torch.tensor(_GROUPED_M_SPLITS, dtype=torch.int64, device="cpu") + + def fn(inp): + return model(inp, m_splits_tensor) + + torch._dynamo.reset() + # Graph-break fallback: numerics must match eager. + compiled = torch.compile(fn) + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + # fullgraph=True: the explicit graph break surfaces the reason. + torch._dynamo.reset() + strict = torch.compile(fn, fullgraph=True) + with pytest.raises(Exception, match="falling back to eager"): + strict(_grouped_input(dtype, device, requires_grad=True)) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_te_grouped_linear_compile_delayed_scaling_falls_back(): + """Delayed-scaling quantizers are not value-opaque; under torch.compile the + module must fall back to eager (a graph break) with eager numerics.""" + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + fp8_recipe = recipe.DelayedScaling() + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS) + + torch._dynamo.reset() + compiled = torch.compile(fn) # no fullgraph: the fallback graph-breaks + # Functional smoke only: delayed scaling advances amax history every call, + # so eager-vs-compiled outputs are not bit-comparable across calls. + try: + inp = _grouped_input(dtype, device, requires_grad=True) + out = compiled(inp) + out.sum().backward() + assert out.shape == (sum(_GROUPED_M_SPLITS), _GROUPED_OUT) + assert inp.grad is not None + for i in range(_GROUPED_NUM_GEMMS): + assert getattr(model, f"weight{i}").grad is not None + finally: + # Drop the delayed-scaling amax-reduction registrations, or every later + # autocast exit would run the (untraceable) fused amax reduction. + te.quantization.FP8GlobalStateManager.reset() + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_te_grouped_linear_compile_save_original_input(): + """torch.compile with ``save_original_input=True``: the op saves the input + by alias and re-splits/re-quantizes it in backward.""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, + _GROUPED_IN, + _GROUPED_OUT, + params_dtype=dtype, + device=device, + save_original_input=True, + ) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + for _ in range(2): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("layout", ["flat", "rank4", "strided"]) +@pytest.mark.parametrize("input_grad,weight_grad", [(True, True), (False, True), (True, False)]) +def test_te_grouped_linear_compile_empty_experts(layout, input_grad, weight_grad): + """Empty experts and non-2D inputs must match native PyTorch gradients.""" + torch._dynamo.reset() + splits = [32, 0, 64, 32] + model = te.GroupedLinear(4, 64, 32, bias=False, params_dtype=torch.bfloat16, device="cuda") + model.requires_grad_(weight_grad) + # Keep the native reference exact across GEMM implementations. + with torch.no_grad(): + for weight in model.parameters(): + weight.copy_(torch.randint_like(weight, -1, 2) / 8) + x = torch.randint(-1, 2, (128, 64), device="cuda").to(torch.bfloat16) / 8 + if layout == "rank4": + x = x.reshape(2, 4, 16, 64) + elif layout == "strided": + x = x.t().contiguous().t() + x.requires_grad_(input_grad) + weights = tuple(model.parameters()) + ref_x = x.detach().clone().requires_grad_(input_grad) + ref_weights = tuple(w.detach().clone().requires_grad_(weight_grad) for w in weights) + ref = torch.cat( + [ + torch.nn.functional.linear(part, weight) + for part, weight in zip(ref_x.reshape(-1, 64).split(splits), ref_weights) + ] + ).reshape(*x.shape[:-1], 32) + out = torch.compile(lambda inp: model(inp, splits), fullgraph=True)(x) + dy = torch.randint_like(out, -1, 2) / 8 + params = ((x,) if input_grad else ()) + (weights if weight_grad else ()) + ref_params = ((ref_x,) if input_grad else ()) + (ref_weights if weight_grad else ()) + actual = out, torch.autograd.grad(out, params, dy) + expected = ref, torch.autograd.grad(ref, ref_params, dy) + torch.testing.assert_close(actual, expected, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +def test_te_grouped_linear_compile_rejects_negative_splits(): + model = te.GroupedLinear(4, 64, 32, params_dtype=torch.bfloat16, device="cuda") + compiled = torch.compile(lambda inp: model(inp, [-8, 136, 0, 0]), fullgraph=True) + with pytest.raises(Exception, match="m_splits entries must be non-negative"): + compiled(_grouped_input(torch.bfloat16, "cuda")) + + +_fused_grouped_cc_ok = torch.cuda.is_available() and ( + (9, 0) <= torch.cuda.get_device_capability() <= (11, 0) +) +_fused_grouped_cublas_ok = _fused_grouped_cc_ok and tex.get_cublasLt_version() >= ( + 130400 if torch.cuda.get_device_capability() < (10, 0) else 130300 +) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif( + not _fused_grouped_cublas_ok, + reason="fused grouped GEMM needs Hopper/Blackwell and a recent cuBLASLt", +) +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize( + "fp8_recipe", + [None] + + ([recipe.Float8CurrentScaling()] if fp8_available else []) + + ([recipe.MXFP8BlockScaling()] if mxfp8_available else []) + + ([recipe.NVFP4BlockScaling(disable_stochastic_rounding=True)] if nvfp4_available else []), + ids=lambda r: "bf16" if r is None else type(r).__name__, +) +@pytest.mark.parametrize("save_original_input", [False, True]) +@pytest.mark.parametrize( + "single_weight,single_bias", + [(False, False), (True, False), (False, True), (True, True)], + ids=["discrete", "grouped-weight", "grouped-bias", "grouped-both"], +) +def test_te_grouped_linear_fused_compiles( + monkeypatch, fp8_recipe, compile_mode, save_original_input, single_weight, single_bias +): + """torch.compile(fullgraph=True) of the fused GroupedTensor path + (``use_grouped_tensor=True``): ``m_splits`` stays a + device tensor, so changing the split distribution in place reuses the very + same graph -- no recompiles, no host sync.""" + monkeypatch.setenv("NVTE_GROUPED_LINEAR_SINGLE_PARAM", "1") + if single_weight and fp8_recipe is not None and fp8_recipe.nvfp4(): + pytest.skip("Native NVFP4 grouped GEMM does not support single grouped weights.") + if fp8_recipe is not None and torch.cuda.get_device_capability() < (10, 0): + if tex.get_cublasLt_version() < 130500: + pytest.skip("FP8 current-scaling fused grouped GEMM on Hopper needs cuBLASLt 13.5+") + dtype = torch.bfloat16 + device = "cuda" + block_scaled = fp8_recipe is not None and not fp8_recipe.float8_current_scaling() + m_splits = [128, 256, 128, 256] if block_scaled else _GROUPED_M_SPLITS + in_features, out_features = (128, 128) if block_scaled else (_GROUPED_IN, _GROUPED_OUT) + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, + in_features, + out_features, + params_dtype=dtype, + device=device, + use_grouped_tensor=True, + save_original_input=save_original_input, + single_grouped_weight=single_weight, + single_grouped_bias=single_bias, + ) + m_splits_dev = torch.tensor(m_splits, dtype=torch.int64, device=device) + + def make_input(): + return torch.randn( + sum(m_splits), in_features, dtype=dtype, device=device, requires_grad=True + ) + + def fn(inp): + if fp8_recipe is None: + return model(inp, m_splits_dev) + with te.autocast(recipe=fp8_recipe): + return model(inp, m_splits_dev) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, make_input(), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + # Two warmup calls absorb the one-time recompile from lazily created + # module attributes. + for _ in range(2): + _assert_close_grouped(fn, compiled, model, make_input()) + baseline = _dynamo_counter("stats", "unique_graphs") + + # Change the split distribution IN PLACE: same graph, no recompile. + changed_splits = ( + [[256, 128, 256, 128], [128, 128, 128, 384]] + if block_scaled + else [[16, 32, 40, 40], [64, 8, 32, 24]] + ) + for splits in changed_splits: + m_splits_dev.copy_(torch.tensor(splits, dtype=torch.int64, device=device)) + _assert_close_grouped(fn, compiled, model, make_input()) + if baseline: + assert ( + _dynamo_counter("stats", "unique_graphs") == baseline + ), "changing device-tensor m_splits values must not recompile" + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not _fused_grouped_cublas_ok, reason="native grouped GEMM unavailable") +@pytest.mark.parametrize( + "fp8_recipe", [None] + ([recipe.MXFP8BlockScaling()] if mxfp8_available else []) +) +@pytest.mark.parametrize("save_original_input", [False, True]) +def test_te_grouped_linear_fused_dynamic_input(fp8_recipe, save_original_input): + model = te.GroupedLinear( + 2, + 128, + 128, + params_dtype=torch.bfloat16, + device="cuda", + use_grouped_tensor=True, + single_grouped_weight=False, + single_grouped_bias=False, + save_original_input=save_original_input, + ) + splits = torch.tensor([128, 128], dtype=torch.int64, device="cuda") + + def fn(inp): + if fp8_recipe is None: + return model(inp, splits) + with te.autocast(recipe=fp8_recipe): + return model(inp, splits) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True, dynamic=True) + for rows in (256, 256, 512, 768): + splits.fill_(rows // 2) + inp = torch.randn(rows, 128, device="cuda", dtype=torch.bfloat16) + _assert_close_grouped(fn, compiled, model, inp) + if rows == 256: + baseline = _dynamo_counter("stats", "unique_graphs") + if baseline: + assert _dynamo_counter("stats", "unique_graphs") == baseline + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not _fused_grouped_cublas_ok, reason="native grouped GEMM unavailable") +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize("primary_quantized", [False, True]) +@pytest.mark.parametrize( + "fp8_recipe", + ([recipe.Float8CurrentScaling()] if fp8_available else []) + + ([recipe.MXFP8BlockScaling()] if mxfp8_available else []), + ids=lambda r: type(r).__name__, +) +def test_te_grouped_linear_grouped_weight_cache( + monkeypatch, compile_mode, primary_quantized, fp8_recipe +): + """Packed primary weights and cached grouped workspaces retain their gradients.""" + torch._dynamo.reset() + monkeypatch.setenv("NVTE_GROUPED_LINEAR_SINGLE_PARAM", "1") + if torch.cuda.get_device_capability() < (10, 0) and tex.get_cublasLt_version() < 130500: + pytest.skip("Current scaling grouped GEMM on Hopper needs cuBLASLt 13.5+") + models = [] + for _ in range(2): + torch.manual_seed(1234) + with te.quantized_model_init(enabled=primary_quantized, recipe=fp8_recipe): + models.append( + te.GroupedLinear( + 2, + 256, + 256, + params_dtype=torch.bfloat16, + device="cuda", + use_grouped_tensor=True, + single_grouped_weight=True, + single_grouped_bias=True, + ) + ) + model, reference = models + splits = torch.tensor([256, 256], device="cuda", dtype=torch.int64) + is_first = None + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, splits, is_first_microbatch=is_first) + + def ref_fn(inp): + with te.autocast(recipe=fp8_recipe): + return reference(inp, splits, is_first_microbatch=is_first) + + def make_input(): + return torch.randn(512, 256, dtype=torch.bfloat16, device="cuda", requires_grad=True) + + fn(make_input()).sum().backward() + model.zero_grad(set_to_none=True) + is_first = True + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, make_input(), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + cached = None + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for is_first in [True, False, False, True]: + _assert_close_grouped(ref_fn, compiled, model, make_input(), ref_model=reference) + if not primary_quantized: + workspace = model._fp8_workspaces["weight"] + if cached is not None: + assert workspace is cached + cached = workspace + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not _fused_grouped_cublas_ok, reason="native grouped GEMM unavailable") +@pytest.mark.parametrize("input_grad,weight_grad", [(True, True), (False, True), (True, False)]) +@pytest.mark.parametrize("layout", ["rank4", "strided"]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_te_grouped_linear_grouped_parameters_native_reference( + monkeypatch, input_grad, weight_grad, layout, dtype +): + """Packed parameter gradients and optimizer updates match independent linears.""" + monkeypatch.setenv("NVTE_GROUPED_LINEAR_SINGLE_PARAM", "1") + torch._dynamo.reset() + splits = [128, 256, 128] + model = te.GroupedLinear( + 3, + 128, + 64, + params_dtype=dtype, + device="cuda", + use_grouped_tensor=True, + single_grouped_weight=True, + single_grouped_bias=True, + ) + model.requires_grad_(weight_grad) + with torch.no_grad(): + for parameter in model.parameters(): + data = parameter.rowwise_data + data.copy_(torch.randint_like(data, -1, 2) / 8) + ref_weight = model.weight.rowwise_data.detach().clone().reshape(3, 64, 128) + ref_bias = model.bias.rowwise_data.detach().clone().reshape(3, 64) + ref_weight.requires_grad_(weight_grad) + ref_bias.requires_grad_(weight_grad) + x = torch.randint(-1, 2, (512, 128), device="cuda").to(dtype) / 8 + x = x.reshape(2, 4, 64, 128) if layout == "rank4" else x.t().contiguous().t() + x.requires_grad_(input_grad) + ref_x = x.detach().clone().requires_grad_(input_grad) + reference = torch.cat( + [ + torch.nn.functional.linear(part, weight, bias) + for part, weight, bias in zip( + ref_x.reshape(-1, 128).split(splits), ref_weight, ref_bias + ) + ] + ).reshape(*x.shape[:-1], 64) + device_splits = torch.tensor(splits, dtype=torch.int64, device="cuda") + output = torch.compile(lambda inp: model(inp, device_splits), fullgraph=True)(x) + dy = torch.randint_like(output, -1, 2) / 8 + output.backward(dy) + reference.backward(dy) + torch.testing.assert_close(output, reference, rtol=0, atol=0) + if input_grad: + torch.testing.assert_close(x.grad, ref_x.grad, rtol=0, atol=0) + if weight_grad: + for param, ref_param in zip(model.parameters(), (ref_weight, ref_bias)): + torch.testing.assert_close(param.grad, ref_param.grad, rtol=0, atol=0) + torch.optim.SGD(model.parameters(), lr=0.125).step() + with torch.no_grad(): + ref_weight.add_(ref_weight.grad, alpha=-0.125) + ref_bias.add_(ref_bias.grad, alpha=-0.125) + torch.testing.assert_close( + model.weight.rowwise_data.reshape_as(ref_weight), ref_weight, rtol=0, atol=0 + ) + torch.testing.assert_close( + model.bias.rowwise_data.reshape_as(ref_bias), ref_bias, rtol=0, atol=0 + ) + + +# --------------------------------------------------------------------------- +# Custom-op list adapters (framework unit test via a toy op) +# --------------------------------------------------------------------------- + + +if _opaque_available: + from dataclasses import dataclass + from typing import List, Optional, Tuple + + from transformer_engine.pytorch.dynamo import register_custom_op_with_autograd + from transformer_engine.pytorch.dynamo.custom_op import SavedTensorRef + from transformer_engine.pytorch.quantized_tensor import restore_from_func_ctx + + @dataclass + class _ToyListFwdArgs: + inp: torch.Tensor + weights: List[Optional[torch.Tensor]] + scales: Tuple[float, ...] + split_metadata: List[int] + + @dataclass + class _ToyListBwdArgs: + grad_output: Optional[torch.Tensor] = None + inp: Optional[torch.Tensor] = None + output: Optional[torch.Tensor] = None + weights: List[Optional[torch.Tensor]] = None + scales: Tuple[float, ...] = () + + def setup_saved_tensors(self, ctx): + self.inp, self.output, *self.weights = restore_from_func_ctx(ctx) + + def _toy_list_saved(args): + return ( + SavedTensorRef("inp"), + SavedTensorRef(None, 0), + *( + SavedTensorRef("weights", i) if w is not None else None + for i, w in enumerate(args.weights) + ), + ) + + def _toy_list_fwd(args): + out = torch.zeros_like(args.inp) + for w, s in zip(args.weights, args.scales): + if w is not None: + out = out + args.inp * w * s + return (out.exp(), _toy_list_saved(args), None) + + def _toy_list_fwd_fake(args): + out = TensorSpec( + shape=tuple(args.inp.shape), + dtype=args.inp.dtype, + requires_grad=True, + device=args.inp.device, + ) + return (out, _toy_list_saved(args), None) + + def _toy_list_setup_ctx(bwd_args, fwd_args, outputs, ctx_attrs, saved): + del outputs, ctx_attrs + bwd_args.scales = fwd_args.scales + return saved + + def _toy_list_bwd(args): + grad = args.grad_output * args.output + dgrad = torch.zeros_like(args.inp) + wgrads = [] + for w, s in zip(args.weights, args.scales): + if w is None: + wgrads.append(None) + continue + dgrad = dgrad + grad * w * s + wgrads.append(grad * args.inp * s) + return (dgrad, wgrads) + + def _toy_list_bwd_fake(args): + spec = lambda t: TensorSpec(shape=tuple(t.shape), dtype=t.dtype, device=t.device) + dgrad = spec(args.inp) + wgrads = [None if w is None else spec(w) for w in args.weights] + return (dgrad, wgrads) + + _toy_list_op = register_custom_op_with_autograd( + op_name="toy_list_op_test", + input_tensors_for_grad=["inp", "weights"], + fwd_arg_type=_ToyListFwdArgs, + fwd_impl=_toy_list_fwd, + fwd_fake_impl=_toy_list_fwd_fake, + setup_context=_toy_list_setup_ctx, + bwd_arg_type=_ToyListBwdArgs, + bwd_impl=_toy_list_bwd, + bwd_fake_impl=_toy_list_bwd_fake, + ) + + _toy_grad_ops = {} + for _grad_fields in [("inp",), ("weights",), ()]: + _indices = tuple(("inp", "weights").index(name) for name in _grad_fields) + _toy_grad_ops[_grad_fields] = register_custom_op_with_autograd( + op_name="toy_grad_returns_" + ("_".join(_grad_fields) or "none"), + input_tensors_for_grad=list(_grad_fields), + fwd_arg_type=_ToyListFwdArgs, + fwd_impl=_toy_list_fwd, + fwd_fake_impl=_toy_list_fwd_fake, + setup_context=_toy_list_setup_ctx, + bwd_arg_type=_ToyListBwdArgs, + bwd_impl=lambda args, indices=_indices: tuple(_toy_list_bwd(args)[i] for i in indices), + bwd_fake_impl=lambda args, indices=_indices: tuple( + _toy_list_bwd_fake(args)[i] for i in indices + ), + ) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("split_metadata", [[], [4, 4]]) +@pytest.mark.parametrize("saved_hooks", [False, True]) +@pytest.mark.parametrize("empty_weights", [False, True]) +def test_custom_op_tensor_list_grads(split_metadata, saved_hooks, empty_weights): + """A toy op with a ``List[Optional[Tensor]]`` input (including a ``None`` + entry riding the sentinel) compiles fullgraph and routes one gradient per + list element.""" + assert _toy_list_op is not None + device = "cuda" + scales = (2.0, 3.0, 0.5) + + def run(fn): + torch.manual_seed(7) + inp = (0.1 * torch.randn(8, 4, device=device)).requires_grad_() + w0 = (0.1 * torch.randn(8, 4, device=device)).requires_grad_() + w2 = (0.1 * torch.randn(8, 4, device=device)).requires_grad_() + hooks = ( + torch.autograd.graph.saved_tensors_hooks(lambda t: t.clone(), lambda t: t) + if saved_hooks + else contextlib.nullcontext() + ) + with hooks: + out = fn(inp, w0, w2) + out.sum().backward() + return out.detach().clone(), inp.grad, w0.grad, w2.grad + + def fn(inp, w0, w2): + args = _ToyListFwdArgs( + inp=inp, + weights=[] if empty_weights else [w0, None, w2], + scales=scales, + split_metadata=split_metadata, + ) + return _toy_list_op(args) + + def ref_fn(inp, w0, w2): + if empty_weights: + return (inp * 0).exp() + return (inp * w0 * scales[0] + inp * w2 * scales[2]).exp() + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + + out_ref, igrad_ref, w0grad_ref, w2grad_ref = run(ref_fn) + out, igrad, w0grad, w2grad = run(compiled) + torch.testing.assert_close(out, out_ref) + torch.testing.assert_close(igrad, igrad_ref) + torch.testing.assert_close(w0grad, w0grad_ref) + torch.testing.assert_close(w2grad, w2grad_ref) + + # Eager through the same op (no compile) must agree too. + out_e, igrad_e, w0grad_e, w2grad_e = run(fn) + torch.testing.assert_close(out_e, out_ref) + torch.testing.assert_close(igrad_e, igrad_ref) + torch.testing.assert_close(w0grad_e, w0grad_ref) + torch.testing.assert_close(w2grad_e, w2grad_ref) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("grad_fields", [("inp",), ("weights",), ()]) +@pytest.mark.parametrize("compile_op", [False, True]) +def test_custom_op_single_gradient_return(grad_fields, compile_op): + """Zero or one gradient field keeps scalar/list schema returns unambiguous.""" + torch._dynamo.reset() + op = _toy_grad_ops[grad_fields] + assert op is not None + inp = torch.randn(8, 4, device="cuda", requires_grad=True) + weight = torch.randn_like(inp, requires_grad=True) + + def fn(x, w): + return op(_ToyListFwdArgs(x, [None, w], (0.0, 0.25), [])) + + reference = (inp * weight * 0.25).exp() + expected = torch.autograd.grad(reference.sum(), (inp, weight)) + actual = (torch.compile(fn, fullgraph=True) if compile_op else fn)(inp, weight) + torch.testing.assert_close(actual, reference) + actual.sum().backward() + for name, tensor, grad in zip(("inp", "weights"), (inp, weight), expected): + if name in grad_fields: + torch.testing.assert_close(tensor.grad, grad) + else: + assert tensor.grad is None + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("changed_tensor", ["inp", "weight", "output"]) +def test_custom_op_saved_refs_version_check(changed_tensor): + inp = torch.randn(4, device="cuda", requires_grad=True) + weight = torch.randn_like(inp, requires_grad=True) + out = _toy_list_op(_ToyListFwdArgs(inp, [weight], (1.0,), [])) + tensor = {"inp": inp, "weight": weight, "output": out}[changed_tensor] + with torch.no_grad(): + tensor.add_(1) + with pytest.raises(RuntimeError, match="modified by an inplace operation"): + out.sum().backward() + + # --------------------------------------------------------------------------- # # transformer_engine.pytorch.ops under torch.compile # --------------------------------------------------------------------------- # @@ -2412,7 +3339,7 @@ def __init__(self, weight=2.0, dtype=torch.float32): self.weight = torch.nn.Parameter(torch.tensor(weight, dtype=dtype, device="cuda")) @classmethod - def forward_compute(cls, args): + def forward_compute(cls, args, *, in_custom_op=False): output = args.input_ * args.weight * args.gain offset = args.offset if isinstance(offset, QuantizedTensorStorage): @@ -2426,7 +3353,7 @@ def forward_compute_fake(cls, args): return args.input_, [()], () @classmethod - def backward_compute(cls, args): + def backward_compute(cls, args, *, in_custom_op=False): dy = args.grad_output return dy * args.weight * args.gain, [((dy * args.input_).sum() * args.gain,)], [()] @@ -2484,7 +3411,7 @@ class _AffinePair(te.ops.FusedOperation): bwd_args_type = _AffinePairBwdArgs @classmethod - def forward_compute(cls, args): + def forward_compute(cls, args, *, in_custom_op=False): intermediate = args.input_ * args.weight0 output = intermediate * args.weight1 + args.residual return output, [(), (intermediate.square(), None)], (intermediate,) @@ -2494,7 +3421,7 @@ def forward_compute_fake(cls, args): return args.input_, [(), (args.input_, None)], (args.input_,) @classmethod - def backward_compute(cls, args): + def backward_compute(cls, args, *, in_custom_op=False): dy = args.grad_output du = dy * args.weight1 + 2 * args.intermediate * args.grad_extra_output return ( @@ -2539,7 +3466,16 @@ def backend(graph, inputs): return torch.compile(fn, fullgraph=True, backend=backend), graphs -def _assert_custom_ops(graphs, name, present=True): +_ExecutionPath = Literal["custom_op", "eager"] + + +def _assert_custom_ops( + graphs, + name, + *, + forward: bool = True, + backward: bool = True, +): targets = { str(node.target).removesuffix(".default").removesuffix("_base") for graph in graphs @@ -2548,9 +3484,7 @@ def _assert_custom_ops(graphs, name, present=True): for node in module.graph.nodes if node.op == "call_function" } - if isinstance(present, bool): - present = (present, present) - for suffix, expected in zip(("", "_backward"), present): + for suffix, expected in (("", forward), ("_backward", backward)): assert (f"transformer_engine_compile.{name}{suffix}" in targets) == expected, targets @@ -2572,32 +3506,79 @@ def _check_ops(fn, model, x, dy, kwargs=None): torch.testing.assert_close(actual, expected) +class _Scenario(NamedTuple): + variant: str + expected_warning: str | None + forward_path: _ExecutionPath + backward_path: _ExecutionPath + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) @pytest.mark.parametrize( - "case,reason,compiled_passes", + "scenario", [ - ("single", None, (True, True)), - ("single_forward", "without a custom op for backward", (True, False)), - ("single_backward", "without a custom op for forward", (False, True)), - ("single_eager", "without a custom op", (False, False)), - ("multi", "several operations", (False, False)), - ("backward_fusion", "backward fusion", (False, False)), - ("legacy", "without a custom op", (False, False)), + _Scenario( + variant="single", + expected_warning=None, + forward_path="custom_op", + backward_path="custom_op", + ), + _Scenario( + variant="single_forward", + expected_warning="without a custom op for backward", + forward_path="custom_op", + backward_path="eager", + ), + _Scenario( + variant="single_backward", + expected_warning="without a custom op for forward", + forward_path="eager", + backward_path="custom_op", + ), + _Scenario( + variant="single_eager", + expected_warning="without a custom op", + forward_path="eager", + backward_path="eager", + ), + # A pipeline can dispatch each operation through its own custom op. + _Scenario( + variant="multi", + expected_warning=None, + forward_path="custom_op", + backward_path="custom_op", + ), + # This test fusion only implements eager backward; forward still compiles. + _Scenario( + variant="backward_fusion", + expected_warning="without a custom op for backward", + forward_path="custom_op", + backward_path="eager", + ), + _Scenario( + variant="legacy", + expected_warning="without a custom op", + forward_path="eager", + backward_path="eager", + ), ], + ids=lambda scenario: scenario.variant, ) -def test_te_ops_pipeline(case, reason, compiled_passes, dtype, monkeypatch): +def test_te_ops_pipeline(scenario, dtype, monkeypatch): torch._dynamo.reset() - if case.startswith("single"): + if scenario.variant.startswith("single"): monkeypatch.setattr( _AffineOp, "compile_ops", tuple( - op if enabled else None - for op, enabled in zip(_AffineOp.compile_ops, compiled_passes) + op if path == "custom_op" else None + for op, path in zip( + _AffineOp.compile_ops, (scenario.forward_path, scenario.backward_path) + ) ), ) - if case == "backward_fusion": + if scenario.variant == "backward_fusion": def fuse(ops, **unused): if len(ops) == 2 and all(isinstance(op, _AffineOp) for op in ops): @@ -2607,23 +3588,35 @@ def fuse(ops, **unused): monkeypatch.setattr(OperationFuser, "backward_fusion_functions", [fuse]) ops = ( [te.ops.Identity()] - if case == "legacy" - else [_AffineOp(float(i + 2), dtype) for i in range(1 if case.startswith("single") else 2)] + if scenario.variant == "legacy" + else [ + _AffineOp(float(i + 2), dtype) + for i in range(1 if scenario.variant.startswith("single") else 2) + ] ) model = te.ops.Sequential(*ops) compiled, graphs = _compile_with_graphs(model) # Keep products exact in BF16 across eager and fused reductions. x = (torch.randint(-8, 9, (8, 16), device="cuda").to(dtype) / 8).requires_grad_() dy = torch.randint_like(x, -8, 9) / 8 - with pytest.warns(UserWarning, match=reason) if reason else contextlib.nullcontext(): + with ( + pytest.warns(UserWarning, match=scenario.expected_warning) + if scenario.expected_warning + else contextlib.nullcontext() + ): _check_ops(compiled, model, x, dy) _check_ops(model, model, x, dy) - _assert_custom_ops(graphs, "_affineop", present=compiled_passes) - if case == "single_forward": + _assert_custom_ops( + graphs, + "_affineop", + forward=scenario.forward_path == "custom_op", + backward=scenario.backward_path == "custom_op", + ) + if scenario.variant == "single_forward": compiled, graphs = _compile_with_graphs(model) with torch.no_grad(): torch.testing.assert_close(compiled(x), x * ops[0].weight) - _assert_custom_ops(graphs, "_affineop", present=(True, False)) + _assert_custom_ops(graphs, "_affineop", backward=False) @pytest.mark.parametrize( @@ -2636,7 +3629,12 @@ def register(**kwargs): registered.append(kwargs["op_name"]) return kwargs["impl"] + def compute(args, *, in_custom_op=False): + return in_custom_op + monkeypatch.setattr("transformer_engine.pytorch.ops.op.register_custom_op", register) + monkeypatch.setattr(_AffineOp, "forward_compute", staticmethod(compute)) + monkeypatch.setattr(_AffineOp, "backward_compute", staticmethod(compute)) monkeypatch.setattr(_AffineOp, "fwd_args_type", _AffineFwdArgs if forward else None) monkeypatch.setattr(_AffineOp, "bwd_args_type", _AffineBwdArgs if backward else None) monkeypatch.setattr(_AffineOp, "compile_ops", (None, None)) @@ -2644,10 +3642,13 @@ def register(**kwargs): assert registered == (["_affineop"] if forward else []) + ( ["_affineop_backward"] if backward else [] ) - assert _AffineOp.compile_ops == ( - _AffineOp.forward_compute if forward else None, - _AffineOp.backward_compute if backward else None, - ) + for custom_op, enabled in zip(_AffineOp.compile_ops, (forward, backward)): + if enabled: + assert custom_op(None) is True + else: + assert custom_op is None + assert _AffineOp.forward_compute(None) is False + assert _AffineOp.backward_compute(None) is False @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @@ -2694,7 +3695,7 @@ def run(input_, extra_input, grad_output, grad_extra_output): ) expected = output, [(), (extra, None)], (dx, [(dw0,), (dw1,)], [(), (dr,)]) torch.testing.assert_close(actual, expected) - _assert_custom_ops(graphs, "_affinepair", present=use_custom_ops) + _assert_custom_ops(graphs, "_affinepair", forward=use_custom_ops, backward=use_custom_ops) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @@ -2719,4 +3720,270 @@ def test_te_ops_forward_kwargs_compile(): with pytest.warns(UserWarning, match="non-tensor keyword arguments"): for gain in (3.0, 5.0): _check_ops(compiled, model, x, dy, {"gain": gain}) - _assert_custom_ops(graphs[-1:], "_affineop", present=False) + _assert_custom_ops(graphs[-1:], "_affineop", forward=False, backward=False) + + +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize("backward", [False, True]) +def test_te_ops_bias_compile_with_fp8_input(backward): + """A quantized tensor subclass must cross the Bias custom-op boundary.""" + torch._dynamo.reset() + quantizer = Float8CurrentScalingQuantizer(fp8_dtype=tex.DType.kFloat8E4M3, device="cuda") + x = quantizer(torch.randn(32, 64, device="cuda", dtype=torch.bfloat16)) + model = te.ops.Bias(64, device="cuda", dtype=torch.bfloat16) + compiled = torch.compile(lambda inp: model(inp), fullgraph=True) + + with contextlib.nullcontext() if backward else torch.no_grad(): + expected = model(x) + actual = compiled(x) + torch.testing.assert_close(actual, expected) + if backward: + dy = torch.randn_like(expected) + expected_db = torch.autograd.grad(expected, model.bias, dy)[0] + actual_db = torch.autograd.grad(actual, model.bias, dy)[0] + torch.testing.assert_close(actual_db, expected_db) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("quantization", [None, "fp8"]) +@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) +@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) +def test_te_ops_linear_bias_compile(dtype, quantization, case, grads, monkeypatch): + _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("backward_override", ["high_precision", "dequantized"]) +def test_te_ops_linear_bias_backward_override(dtype, backward_override, monkeypatch): + _check_linear_bias_compile( + dtype, "fp8", "pair", "all", monkeypatch, backward_override=backward_override + ) + + +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize( + "input_dtype,weight_dtype", + [(torch.float32, torch.bfloat16), (torch.bfloat16, torch.float32)], +) +def test_te_ops_linear_saved_fp8_dtype_with_autocast(input_dtype, weight_dtype): + """Saved FP8 tensors retain their source dtype across the custom-op boundary.""" + torch._dynamo.reset() + model = te.ops.BasicLinear(32, 64, device="cuda", dtype=weight_dtype) + eager_model = copy.deepcopy(model) + fp8_recipe = recipe.Float8CurrentScaling() + + def run(module, inp): + with torch.autocast(device_type="cuda", dtype=torch.bfloat16): + with te.autocast(recipe=fp8_recipe): + return module(inp) + + compiled, graphs = _compile_with_graphs(lambda inp: run(model, inp)) + x = torch.randn(16, 32, device="cuda", dtype=input_dtype, requires_grad=True) + eager_x = x.detach().clone().requires_grad_() + actual = compiled(x) + expected = run(eager_model, eager_x) + torch.testing.assert_close(actual, expected) + + dy = torch.randn_like(actual) + actual_grads = torch.autograd.grad(actual, (x, model.weight), dy) + expected_grads = torch.autograd.grad(expected, (eager_x, eager_model.weight), dy) + torch.testing.assert_close(actual_grads, expected_grads) + _assert_custom_ops(graphs, "basiclinear") + + with torch.autocast(device_type="cuda", dtype=torch.bfloat16): + with te.autocast(recipe=fp8_recipe): + args = model.pack_forward_args( + [OperationContext()], + x, + prev_op_grad_output_quantizer=None, + next_op_input_quantizer=None, + basic_op_kwargs=[{}], + ) + _, _, (saved_input, saved_weight) = model.forward_compute_fake(args) + assert saved_input.dtype == x.dtype + assert saved_weight.dtype == model.weight.dtype + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_te_ops_linear_workspace_fake_mode(monkeypatch): + def unexpected_allocation(*args, **kwargs): + pytest.fail("Fake initialization must not populate the real workspace cache") + + monkeypatch.setattr( + "transformer_engine.pytorch.ops.basic.basic_linear.get_cublas_workspace", + unexpected_allocation, + ) + with FakeTensorMode(): + te.ops.BasicLinear(32, 64, device="cuda", dtype=torch.bfloat16) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("training", [False, True]) +@pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) +def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device): + """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" + script = textwrap.dedent( + """ + import contextlib + import sys + import torch + import transformer_engine.pytorch as te + from transformer_engine.pytorch.cpp_extensions.gemm import get_cublas_workspace + from torch._dynamo.utils import counters + + training, initial_device = sys.argv[1:] + training = training == "True" + assert get_cublas_workspace.cache_info().currsize == 0 + linear = te.ops.BasicLinear(32, 64, device=initial_device, dtype=torch.bfloat16) + if initial_device == "cpu": + assert get_cublas_workspace.cache_info().currsize == 0 + linear.cuda() + elif initial_device == "meta": + assert get_cublas_workspace.cache_info().currsize == 0 + linear.to_empty(device="cuda") + linear.reset_parameters() + assert get_cublas_workspace.cache_info().currsize > 0 + model = te.ops.Sequential(linear, te.ops.Bias(64, dtype=torch.bfloat16)) + x = torch.randn(32, 32, device="cuda", dtype=torch.bfloat16, requires_grad=training) + targets = (x, *model.parameters()) + dy = torch.randn(32, 64, device="cuda", dtype=torch.bfloat16) + + def forward(x): + return model(x) + + compiled = torch.compile(forward, fullgraph=True, mode="reduce-overhead") + counters.clear() + with contextlib.nullcontext() if training else torch.no_grad(): + for _ in range(5): + torch.compiler.cudagraph_mark_step_begin() + actual = compiled(x) + if training: + grads = torch.autograd.grad(actual, targets, dy) + # Eager reference runs only after the first compiled invocation. + expected = forward(x) + torch.testing.assert_close(actual, expected) + if training: + expected_grads = torch.autograd.grad(expected, targets, dy) + torch.testing.assert_close(grads, expected_grads) + del grads, expected_grads + del actual, expected + torch.cuda.synchronize() + assert not counters["inductor"]["cudagraph_skips"], counters["inductor"] + assert counters["inductor"]["cudagraph_recorded_non_static_inputs"] > 0, dict( + counters["inductor"] + ) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script, str(training), initial_device], + capture_output=True, + text=True, + check=False, + timeout=180, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def _check_linear_bias_compile( + dtype, quantization, case, grads, monkeypatch, *, backward_override=None +): + if quantization == "fp8" and not fp8_available: + pytest.skip(reason_for_no_fp8) + if (case == "linear" and grads == "bias") or (case == "bias" and grads == "weight"): + pytest.skip("Requested parameter is absent") + torch._dynamo.reset() + if case == "unfused": + monkeypatch.setattr(OperationFuser, "forward_fusion_functions", []) + ops = [] + if case != "bias": + ops.append(te.ops.BasicLinear(32, 64, dtype=dtype)) + ops[-1].weight.requires_grad_(grads in ("all", "weight")) + if case != "linear": + ops.append(te.ops.Bias(32 if case == "bias" else 64, dtype=dtype)) + ops[-1].bias.requires_grad_(grads in ("all", "bias")) + model = te.ops.Sequential(*ops) + eager_model = copy.deepcopy(model) + quant_recipe = ( + recipe.Float8CurrentScaling(backward_override=backward_override) + if quantization == "fp8" + else None + ) + + def run(x, module=model): + if quant_recipe is None: + return module(x) + with te.autocast(recipe=quant_recipe): + return module(x) + + compiled, graphs = _compile_with_graphs(run) + x = torch.randn(2, 16, 32, device="cuda", dtype=dtype) + x.requires_grad_(grads in ("all", "input")) + targets = tuple(t for t in (x, *model.parameters()) if t.requires_grad) + eager_targets = tuple(t for t in (x, *eager_model.parameters()) if t.requires_grad) + graph_count = None + with torch.no_grad() if grads == "none" else contextlib.nullcontext(): + for iteration in range(3): + with torch.no_grad(): + for tensor in (x, *model.parameters()): + if iteration == 0: + # These binary fractions survive FP8 quantization exactly, + # allowing a strict comparison with native PyTorch. + tensor.copy_(torch.randint(-2, 3, tensor.shape, device=tensor.device) / 16) + else: + tensor.uniform_(-0.5, 0.5) + for param, eager_param in zip(model.parameters(), eager_model.parameters()): + eager_param.copy_(param) + actual = compiled(x) + expected = run(x, eager_model) + torch.testing.assert_close(actual, expected) + if iteration == 0: + reference = x.double() + for op in ops: + reference = ( + torch.nn.functional.linear(reference, op.weight.double()) + if isinstance(op, BasicLinear) + else reference + op.bias.double() + ) + torch.testing.assert_close( + actual, reference, check_dtype=False, **dtype_tols(dtype) + ) + if targets: + dy = ( + torch.randint_like(actual, -2, 3) / 16 + if iteration == 0 + else torch.randn_like(actual) + ) + actual_grads = torch.autograd.grad(actual, targets, dy) + torch.testing.assert_close( + actual_grads, torch.autograd.grad(expected, eager_targets, dy) + ) + if iteration == 0: + torch.testing.assert_close( + actual_grads, + torch.autograd.grad(reference, targets, dy.double()), + **dtype_tols(dtype), + ) + if iteration == 1: + graph_count = len(graphs) + elif iteration == 2: + assert len(graphs) == graph_count + _assert_custom_ops( + graphs, + "forwardlinearbiasactivation", + forward=case == "pair", + backward=False, + ) + _assert_custom_ops( + graphs, + "basiclinear", + forward=case in ("linear", "unfused"), + backward=case != "bias" and grads in ("all", "input", "weight"), + ) + _assert_custom_ops( + graphs, + "bias", + forward=case in ("bias", "unfused"), + backward=case != "linear" and bool(targets), + ) diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index 843ec412c3d..077a9a2b3a1 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -363,6 +363,7 @@ list(APPEND transformer_engine_cuda_arch_specific_sources cast/cast_grouped.cu cast/cast_grouped_dbias.cu cast/fused_group_requantize.cu + fused_rope/fused_mla_kv_rope_mxfp8.cu gemm/cutlass_grouped_gemm.cu hadamard_transform/group_hadamard_transform.cu hadamard_transform/graph_safe_group_hadamard_transform.cu @@ -373,6 +374,7 @@ list(APPEND transformer_engine_cuda_arch_specific_sources hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu multi_tensor/compute_scale.cu + normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu recipe/mxfp8_scaling.cu recipe/nvfp4.cu transpose/quantize_transpose_square_blockwise.cu diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index a919b0a5eec..6b3fd43f001 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -173,7 +173,8 @@ __global__ void __launch_bounds__(THREADS_PER_WARP) __device__ __forceinline__ void fence_acquire_tensormap(const CUtensorMap *tensor_map) { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) // The descriptor updater and consumer execute in different CTAs, so CTA scope is insufficient. - asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], 128;" ::"l"(tensor_map)); + asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], 128;" ::"l"(tensor_map) + : "memory"); #else NVTE_DEVICE_ERROR("fence_acquire_tensormap is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) diff --git a/transformer_engine/common/cast/dispatch/gated.cuh b/transformer_engine/common/cast/dispatch/gated.cuh index 06e8f0e306c..0c1761264a1 100644 --- a/transformer_engine/common/cast/dispatch/gated.cuh +++ b/transformer_engine/common/cast/dispatch/gated.cuh @@ -46,7 +46,10 @@ void quantize_gated_fwd_helper(const NVTETensor nvte_input, NVTETensor nvte_outp switch (output->scaling_mode) { case NVTE_DELAYED_TENSOR_SCALING: { - const bool use_tma_kernels = (cols % 32 == 0) && is_supported_by_CC_100(); + const bool use_tma_kernels = + (cols % 32 == 0) && is_supported_by_CC_100() && + fp8::cast_gated_tma_fits_device( + input.dtype(), output->dtype()); if (use_tma_kernels) { Tensor dummy_grad_tensor; fp8::cast_gated_tma(input, dummy_grad_tensor, @@ -137,7 +140,10 @@ void quantize_gated_bwd_helper(const NVTETensor nvte_grad, const NVTETensor nvte switch (output->scaling_mode) { case NVTE_DELAYED_TENSOR_SCALING: { - const bool use_tma_kernels = (cols % 32 == 0) && is_supported_by_CC_100(); + const bool use_tma_kernels = + (cols % 32 == 0) && is_supported_by_CC_100() && + fp8::cast_gated_tma_fits_device( + gated_input.dtype(), output->dtype()); if (use_tma_kernels) { fp8::cast_gated_tma(gated_input, grad, output, p, stream); diff --git a/transformer_engine/common/cast/fp8/gated_fp8.cuh b/transformer_engine/common/cast/fp8/gated_fp8.cuh index 631143c6aef..08583f6298d 100644 --- a/transformer_engine/common/cast/fp8/gated_fp8.cuh +++ b/transformer_engine/common/cast/fp8/gated_fp8.cuh @@ -17,6 +17,7 @@ #include #include "../../common.h" +#include "../../util/cuda_runtime.h" #include "../../util/math.h" #include "../../util/ptx.cuh" #include "../../util/vectorized_pointwise.h" @@ -247,8 +248,15 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) // Wait for TMA transfer to have finished reading shared memory. ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_it < ITERATIONS && next_it >= BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); if (amax_ptr != nullptr) { @@ -279,6 +287,45 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) } } // namespace kernel +// Dynamic shared memory requested by cast_fp8_gated_kernel. Must match the +// buffer layout inside the kernel. +inline size_t cast_gated_tma_dynamic_shmem_size(const bool is_bwd, const DType itype, + const DType otype) { + using namespace kernel; + const size_t buff_elems_total = BUFFERS_NUM * SHMEM_DIM_Y * SHMEM_DIM_X; + const size_t buff_size_aligned_in = + DIVUP_TO_MULTIPLE(buff_elems_total * typeToNumBits(itype) / 8, TMA_SHMEM_ALIGNMENT); + const size_t buff_size_aligned_out = + DIVUP_TO_MULTIPLE(buff_elems_total * typeToNumBits(otype) / 8, TMA_SHMEM_ALIGNMENT); + const size_t grad_mem = (is_bwd ? buff_size_aligned_in : 0); + const size_t in_act_mem = buff_size_aligned_in; + const size_t in_gate_mem = buff_size_aligned_in; + const size_t out_act_mem = buff_size_aligned_out; + const size_t out_gate_mem = buff_size_aligned_out; + return grad_mem + (in_act_mem + in_gate_mem) + (out_act_mem + out_gate_mem) + TMA_SHMEM_ALIGNMENT; +} + +// Whether cast_fp8_gated_kernel fits in the shared memory of the current device. +// Devices with compute capability 10.0+ differ in shared memory per block +// (e.g. SM 12.0 has much less than SM 10.0), so FP32 configurations may not fit. +// The static shared memory is read from the compiled kernel so that __shared__ +// arrays in the device functions it calls (e.g. reduce_max) are included. +template +bool cast_gated_tma_fits_device(const DType itype, const DType otype) { + using namespace kernel; + size_t static_shmem_size = 0; + TRANSFORMER_ENGINE_TYPE_SWITCH_INPUT( + itype, IType, + TRANSFORMER_ENGINE_TYPE_SWITCH_OUTPUT( + otype, OType, + static_shmem_size = cuda::static_shared_memory_size(reinterpret_cast( + &cast_fp8_gated_kernel)););); + const size_t required = + cast_gated_tma_dynamic_shmem_size(IS_BWD, itype, otype) + static_shmem_size; + return required <= cuda::max_shared_memory_per_block_optin(); +} + template void cast_gated_tma(const Tensor &gated_input, const Tensor &grad, Tensor *output, ParamOP &p, @@ -329,19 +376,8 @@ void cast_gated_tma(const Tensor &gated_input, const Tensor &grad, Tensor *outpu SHMEM_DIM_X, tensor_stride_elems, cols, typeToNumBits(output->dtype())); - const size_t buff_elems_total = BUFFERS_NUM * SHMEM_DIM_Y * SHMEM_DIM_X; - const size_t buff_size_aligned_in = - DIVUP_TO_MULTIPLE(buff_elems_total * sizeof(IType), TMA_SHMEM_ALIGNMENT); - const size_t buff_size_aligned_out = - DIVUP_TO_MULTIPLE(buff_elems_total * sizeof(OType), TMA_SHMEM_ALIGNMENT); - const size_t grad_mem = (IS_BWD ? buff_size_aligned_in : 0); - const size_t in_act_mem = buff_size_aligned_in; - const size_t in_gate_mem = buff_size_aligned_in; - const size_t out_act_mem = buff_size_aligned_out; - const size_t out_gate_mem = buff_size_aligned_out; - - const size_t shmem_size = grad_mem + (in_act_mem + in_gate_mem) + - (out_act_mem + out_gate_mem) + TMA_SHMEM_ALIGNMENT; + const size_t shmem_size = + cast_gated_tma_dynamic_shmem_size(IS_BWD, gated_input.dtype(), output->dtype()); auto kernel = cast_fp8_gated_kernel; NVTE_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, diff --git a/transformer_engine/common/cast/fp8/quantize_fp8.cuh b/transformer_engine/common/cast/fp8/quantize_fp8.cuh index 9b1ebf5c040..1f6e33e441a 100644 --- a/transformer_engine/common/cast/fp8/quantize_fp8.cuh +++ b/transformer_engine/common/cast/fp8/quantize_fp8.cuh @@ -198,10 +198,17 @@ __global__ void __launch_bounds__(FP8_THREADS_PER_CHUNK) ptx::cp_async_bulk_commit_group(); // Wait for TMA transfer to have finished reading shared memory. - ptx::cp_async_bulk_wait_group_read(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (iter + 1 < FP8_ITERATIONS && iter + 1 >= FP8_BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); parity ^= 1; @@ -331,10 +338,17 @@ __global__ void __launch_bounds__(THREADS_PER_BLOCK) ptx::cp_async_bulk_commit_group(); // Wait for TMA transfer to have finished reading shared memory. - ptx::cp_async_bulk_wait_group_read<1>(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_iter < ITERATIONS && next_iter >= SHMEM_BUFFERS) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); if (amax_ptr != nullptr) { diff --git a/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh b/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh index 31feaf833dc..bf73e935014 100644 --- a/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh +++ b/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh @@ -341,8 +341,9 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 4) group_block_scaled_2d_tma static_cast(global_col_base), static_cast(global_row_base), &tma_mbar); } ptx::mbarrier_wait_parity(&tma_mbar, 0); - if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); + // Every thread must finish waiting before the leader invalidates the barrier. __syncthreads(); + if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); // ---- Optional dbias: per-tile column sum of the high-precision input (smem-resident) ---- if (dbias_workspace != nullptr) { @@ -628,8 +629,9 @@ __global__ void __launch_bounds__(kThreadsPerBlock) group_block_scaled_1d_tma_ke static_cast(global_col_base), static_cast(global_row_base), &tma_mbar); } ptx::mbarrier_wait_parity(&tma_mbar, 0); - if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); + // Every thread must finish waiting before the leader invalidates the barrier. __syncthreads(); + if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); // ---- Optional dbias: per-tile column sum of the high-precision input (smem-resident) ---- if (dbias_workspace != nullptr) { diff --git a/transformer_engine/common/cast/fused_group_requantize.cu b/transformer_engine/common/cast/fused_group_requantize.cu index a2156f369a9..aad83cb67c0 100644 --- a/transformer_engine/common/cast/fused_group_requantize.cu +++ b/transformer_engine/common/cast/fused_group_requantize.cu @@ -13,10 +13,12 @@ #include #include +#include #include #include #include "../common.h" +#include "../util/cuda_runtime.h" #include "../util/ptx_arch_spec.cuh" #include "../utils.cuh" #include "mxfp8/swizzle.cuh" @@ -380,7 +382,7 @@ __global__ void __launch_bounds__(kThreads) reinterpret_cast(&output_tensor_map), col_base, row_base, reinterpret_cast(quantized)); ptx::cp_async_bulk_commit_group(); - ptx::cp_async_bulk_wait_group_read<0>(); + ptx::cp_async_bulk_wait_group(); } __syncthreads(); #else @@ -413,6 +415,172 @@ void launch_fused_group_requantize(const Tensor &input, Tensor *output, input_scale_stride); } +// --------------------------------------------------------------------------- +// Wide-tile variant for the fast-math case without a dequantized output. +// +// Same per-element arithmetic as fused_group_requantize_kernel, so the +// outputs are bit-identical; only the work decomposition differs. The kernel above runs one 32x128 +// tile per 128-thread CTA, thread 0 binary-searches the group offsets before the tile can finish, +// and both scale outputs are written one byte at a time. Here a CTA owns a 128x128 tile: warp w +// takes rows 32w..32w+31 and lane l columns 4l..4l+3, so the data moves as coalesced 128-byte row +// segments held in registers, and each of the two swizzled scale outputs is one contiguous +// 512-byte block per tile. The grid is persistent and walks the tiles in row-block order, so a +// CTA stops at the first tile past tensor_offsets[num_groups]; a capacity-sized buffer with a +// short live prefix then costs little more than the live rows. + +constexpr int kWideTile = 128; +constexpr int kWideThreads = 128; +constexpr int kWideMaxGroups = 128; +constexpr int kWideCtasPerSM = 16; + +template +__global__ void __launch_bounds__(kWideThreads) + fused_group_requantize_wide_kernel(const IType *const input, + const e8m0_t *const input_scale_inv, + const int input_scale_stride, fp8e4m3 *const colwise_out, + e8m0_t *const rowwise_scale_inv_swizzled, + e8m0_t *const colwise_scale_inv, + const int64_t *const tensor_offsets, const int num_groups, + const int num_cols, const int num_tiles) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + static_assert(sizeof(IType) == 1); + __shared__ int64_t offsets[kWideMaxGroups + 1]; + __shared__ alignas(16) e8m0_t colwise_scale_block[kWideTile * 4]; + + const int tid = threadIdx.x; + const int warp = tid / THREADS_PER_WARP; + const int lane = tid % THREADS_PER_WARP; + const int tiles_x = num_cols / kWideTile; + const int words_per_row = num_cols / 4; + // This lane's 4 columns lie in the 32-column scaling block lane / 8 of the tile. + const int scale_shift = 8 * (lane / 8); + for (int i = tid; i <= num_groups; i += kWideThreads) { + offsets[i] = tensor_offsets[i]; + } + __syncthreads(); + const int64_t live_end = offsets[num_groups]; + + const uint32_t *const input_words = reinterpret_cast(input); + uint32_t *const output_words = reinterpret_cast(colwise_out); + for (int tile = blockIdx.x; tile < num_tiles; tile += gridDim.x) { + const int tile_x = tile % tiles_x; + const int tile_y = tile / tiles_x; + const int row_base = tile_y * kWideTile; + const int col_base = tile_x * kWideTile; + const int64_t row_element_base = static_cast(row_base) * num_cols; + // Rows at or past the live end are left untouched, as by the kernel above. Later tiles of + // this CTA lie further down, so the walk ends here. + if (row_element_base >= live_end) { + break; + } + + // 32 rows x 4 columns per thread, one 4-byte word per row. + const int row0 = row_base + 32 * warp; + const uint32_t *const src = + input_words + static_cast(row0) * words_per_row + col_base / 4 + lane; + uint32_t data[32]; +#pragma unroll + for (int r = 0; r < 32; ++r) { + data[r] = __ldg(src + static_cast(r) * words_per_row); + } + // Lane l holds the 4 rowwise scale codes (the tile's four 32-column blocks) of row row0 + l. + const uint32_t scale_word = __ldg(reinterpret_cast( + input_scale_inv + static_cast(row0 + lane) * input_scale_stride + + col_base / MXFP8_SCALE_DIM)); + + // The owning group, as found by the upper_bound lookup of the kernel above. 128-aligned group + // sizes mean a tile never straddles a group boundary. + int group = 0; + for (int i = 0; i + 1 < num_groups; ++i) { + group += (offsets[i + 1] <= row_element_base) ? 1 : 0; + } + const int group_start = static_cast(offsets[group] / num_cols); + const int group_rows = static_cast((offsets[group + 1] - offsets[group]) / num_cols); + + // GEMM-swizzled copy of the rowwise scales: the tile's 128 rows x 4 codes are one 512-byte + // block, in which row r of the tile starts at (r % 32) * 16 + (r / 32) * 4. + *reinterpret_cast(rowwise_scale_inv_swizzled + + (static_cast(tile_y) * tiles_x + tile_x) * 512 + + lane * 16 + warp * 4) = scale_word; + + // Dequantize to BF16 and take the amax of each column over the 32-row block. + ptx::bf16x2 dequantized01[32]; + ptx::bf16x2 dequantized23[32]; + ptx::bf16x2 amax01 = {static_cast(0.0f), static_cast(0.0f)}; + ptx::bf16x2 amax23 = {static_cast(0.0f), static_cast(0.0f)}; +#pragma unroll + for (int r = 0; r < 32; ++r) { + const e8m0_t scale_code = + static_cast((__shfl_sync(0xffffffff, scale_word, r) >> scale_shift) & 0xFF); + const uint16_t values01 = static_cast(data[r]); + const uint16_t values23 = static_cast(data[r] >> 16); + dequantized01[r] = + dequantize_mxfp8_2x(reinterpret_cast &>(values01), scale_code); + dequantized23[r] = + dequantize_mxfp8_2x(reinterpret_cast &>(values23), scale_code); + ptx::abs_max_2x(amax01, amax01, dequantized01[r]); + ptx::abs_max_2x(amax23, amax23, dequantized23[r]); + } + constexpr float kMaxNormRcp = Quantized_Limits::max_norm_rcp; + const e8m0_t exponent0 = ptx::float_to_e8m0(static_cast(amax01.x) * kMaxNormRcp); + const e8m0_t exponent1 = ptx::float_to_e8m0(static_cast(amax01.y) * kMaxNormRcp); + const e8m0_t exponent2 = ptx::float_to_e8m0(static_cast(amax23.x) * kMaxNormRcp); + const e8m0_t exponent3 = ptx::float_to_e8m0(static_cast(amax23.y) * kMaxNormRcp); + const ptx::bf16x2 multiplier01 = {ptx::exp2f_rcp(exponent0), + ptx::exp2f_rcp(exponent1)}; + const ptx::bf16x2 multiplier23 = {ptx::exp2f_rcp(exponent2), + ptx::exp2f_rcp(exponent3)}; + + uint32_t *const dst = + output_words + static_cast(row0) * words_per_row + col_base / 4 + lane; +#pragma unroll + for (int r = 0; r < 32; ++r) { + ptx::fp8e4m3x4 quantized; + ptx::mul_cvt_4x(quantized, dequantized01[r], multiplier01, dequantized23[r], multiplier23); + dst[static_cast(r) * words_per_row] = reinterpret_cast(quantized); + } + + // Columnwise scales: column c of the tile and 32-row block w go to (c % 32) * 16 + (c / 32) * 4 + // + w of the tile's 512-byte block in the group's swizzled layout. Stage the block in shared + // memory, then store it with one 4-byte word per thread. + const e8m0_t exponents[4] = {exponent0, exponent1, exponent2, exponent3}; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int c = 4 * lane + j; + colwise_scale_block[(c % 32) * 16 + (c / 32) * 4 + warp] = exponents[j]; + } + __syncthreads(); + const size_t colwise_scale_base = static_cast(group_start) / MXFP8_SCALE_DIM * num_cols; + const int tile_in_group = (row_base - group_start) / kWideTile; + const int colwise_scale_tiles_x = group_rows / kWideTile; + e8m0_t *const colwise_scale_dst = + colwise_scale_inv + colwise_scale_base + + (static_cast(tile_x) * colwise_scale_tiles_x + tile_in_group) * 512; + reinterpret_cast(colwise_scale_dst)[tid] = + reinterpret_cast(colwise_scale_block)[tid]; + __syncthreads(); // The next tile rewrites colwise_scale_block. + } +#else + NVTE_DEVICE_THREAD0_ERROR("Fused grouped requantization requires Blackwell (SM100+) hardware."); +#endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +} + +template +void launch_fused_group_requantize_wide(const Tensor &input, Tensor *output, + const Tensor &tensor_offsets, const int num_groups, + const int num_tiles, const int num_cols, + const int input_scale_stride, cudaStream_t stream) { + const int grid = static_cast( + std::min(num_tiles, static_cast(cuda::sm_count()) * kWideCtasPerSM)); + fused_group_requantize_wide_kernel<<>>( + reinterpret_cast(input.data.dptr), + reinterpret_cast(input.scale_inv.dptr), input_scale_stride, + reinterpret_cast(output->columnwise_data.dptr), + reinterpret_cast(output->scale_inv.dptr), + reinterpret_cast(output->columnwise_scale_inv.dptr), + reinterpret_cast(tensor_offsets.data.dptr), num_groups, num_cols, num_tiles); +} + void fused_group_requantize(const Tensor &input, Tensor *output, const Tensor &tensor_offsets, Tensor *dequantized, const QuantizationConfig *quant_config, cudaStream_t stream) { @@ -515,11 +683,30 @@ void fused_group_requantize(const Tensor &input, Tensor *output, const Tensor &t return; } + const bool use_fast_math = quant_config != nullptr && quant_config->use_fast_math; + + // The wide-tile kernel covers the fast-math case without a dequantized output. It loads the + // rowwise scales and stores both scale outputs as 4-byte words. + const int64_t num_wide_tiles = + static_cast(num_rows / kWideTile) * static_cast(num_cols / kWideTile); + const bool use_wide_kernel = + use_fast_math && !return_dequantized && num_groups <= kWideMaxGroups && + num_wide_tiles <= static_cast(kMaxInt) && input_scale_stride % 4 == 0 && + is_aligned_ptr(input.scale_inv.dptr, 4) && is_aligned_ptr(output->scale_inv.dptr, 4) && + is_aligned_ptr(output->columnwise_scale_inv.dptr, 4); + if (use_wide_kernel) { + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( + input.data.dtype, IType, + launch_fused_group_requantize_wide(input, output, tensor_offsets, num_groups, + static_cast(num_wide_tiles), num_cols, + input_scale_stride, stream);); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); + return; + } + alignas(64) CUtensorMap output_tensor_map{}; create_2D_tensor_map(output_tensor_map, output->columnwise_data, num_rows, num_cols, kTileRows, kTileCols, num_cols, 0, typeToNumBits(output->columnwise_data.dtype)); - - const bool use_fast_math = quant_config != nullptr && quant_config->use_fast_math; TRANSFORMER_ENGINE_SWITCH_CONDITION( use_fast_math, USE_FAST_MATH, TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( diff --git a/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh index 1face261bd6..91af8b139d9 100644 --- a/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh @@ -213,10 +213,17 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) ptx::cp_async_bulk_commit_group(); // Wait for TMA transfer to have finished reading shared memory. - ptx::cp_async_bulk_wait_group_read<1>(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_iter < ITERATIONS && next_iter >= BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); parity ^= 1; diff --git a/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh index 44dea1a7c76..31e445d4fb4 100644 --- a/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh @@ -74,6 +74,8 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) constexpr size_t STAGES = CHUNK_DIM_Y / BUFF_DIM_Y; static_assert(STAGES >= 1); + static_assert(STAGES <= BUFFS_NUM, + "Output-buffer reuse requires a TMA completion handoff to the CTA"); constexpr bool IS_CACHED_ACT_OP = ROWWISE_SCALING && COLWISE_SCALING; constexpr bool ONLY_COLWISE_SCALING = COLWISE_SCALING && (!ROWWISE_SCALING); @@ -195,10 +197,6 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) const size_t stage_offset_Y = stage * BUFF_DIM_Y; if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); - const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -704,6 +702,13 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) } parity ^= 1; + + // Ensure all S2G operations issued by the master have completed before the CTA exits. + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } // NOLINT(readability/fn_size) diff --git a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh index 141efabbaf5..fcf8819e5f8 100644 --- a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh @@ -359,10 +359,17 @@ __global__ void __launch_bounds__(128) chunk_it_offset_y, reinterpret_cast(&out_sh[buff])); ptx::cp_async_bulk_commit_group(); - ptx::cp_async_bulk_wait_group_read<1>(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_iter < ITERATIONS && next_iter >= BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); destroy_barriers(mbar, is_master_thread); diff --git a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh index bd8c3052f60..08605a27867 100644 --- a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh @@ -882,7 +882,13 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ DIVUP_TO_MULTIPLE(DIVUP(cols, static_cast(SCALE_DIM_X)), scale_alignment_X_rowwise); const size_t scale_stride_colwise = DIVUP_TO_MULTIPLE(cols, scale_alignment_X_colwise); - const size_t tensor_base_for_scales = is_single_tensor ? tensor_start_offset : tensor_base; + // When is single tensor: + // - tensor_base_for_scales: offset of this tensor in the whole grouped tensor + // - scales_rowwise & scales_colwise: points to the scale of the whole grouped tensor + // When not single tensor: + // - tensor_base_for_scales: 0 + // - scales_rowwise & scales_colwise: points to the scale of this tensor + const size_t tensor_base_for_scales = is_single_tensor ? tensor_start_offset : 0; e8m0_t *const scales_rowwise = scales_rowwise_ptr + (is_single_tensor ? 0 : tensor_base / SCALE_DIM_X); e8m0_t *const scales_colwise = diff --git a/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh index 878fc93107c..0dab09f163e 100644 --- a/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh @@ -344,11 +344,11 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) group_scaled_swiglu_mxfp8_k ptx::mbarrier_wait_parity_acquire_cta_shared_cta(&IN_buff_readable_mbar[buff_in], IN_buff_readable_parity[buff_in]); IN_buff_readable_parity[buff_in] ^= 1; - // Wait until the store groups still holding an output slice have drained. Only - // the leading thread commits those groups, so the wait is a no-op on the other - // threads and the barrier is what stops them from overwriting a slice the TMA - // unit has not finished reading. - ptx::cp_async_bulk_wait_group_read(); + // Only the issuing thread can wait for the TMA read of the output slice. + // Hand its completion off to all cooperative writers before reusing the slice. + if (leading_thread) { + ptx::cp_async_bulk_wait_group_read(); + } __syncthreads(); const size_t buff = buff_in; @@ -379,6 +379,12 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) group_scaled_swiglu_mxfp8_k total_work_blocks, work_blocks_X); } + // The last stage has no following iteration to wait for its TMA store. + if (leading_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(IN_buff_readable_mbar, leading_thread); } // if constexpr (is_single_tensor) #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) diff --git a/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh index f7749f65dd7..e04a72e5d9d 100644 --- a/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh @@ -206,11 +206,17 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) const size_t next_stage = stage + 1; const size_t stage_offset_Y = stage * BUFF_DIM_Y; - if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); + // S2G bulk async-groups are thread-local, so only the issuing thread can wait for the + // previous TMA read of this output buffer to complete. Hand that completion off to the + // rest of the CTA before any thread overwrites the reused buffer. + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + __syncthreads(); + } + if (next_stage < STAGES) { const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -633,6 +639,12 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) atomicMaxFloat(amax_ptr, block_amax); } + // Ensure all S2G operations issued by the master have completed before the CTA exits. + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } diff --git a/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh index 2aad1aae165..baf9e5a6cec 100644 --- a/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh @@ -1027,8 +1027,8 @@ __global__ void quantize_mxfp8_kernel_cast_only( ptx::cp_async_bulk_wait_group_read(); ptx::mbarrier_arrive_expect_tx(&stg_consumer[read_state.index()], 0u); } + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); } else { PipeState read_state; @@ -1510,6 +1510,17 @@ __global__ void quantize_mxfp8_kernel_cast_only( } #pragma unroll 1 for (int32_t iter = 0; iter < CastTraits::iterLayout::num; iter++) { + // S2G bulk async-groups are thread-local. Once the output ring wraps, the issuing + // thread must observe completion and hand it off to the CTA before cooperative STS. + if constexpr (CastTraits::_need_wait_group) { + if (iter >= CastTraits::numStages) { + if (warpId == 0 && leader) { + ptx::cp_async_bulk_wait_group_read(); + } + __syncthreads(); + } + } + { int32_t next = iter + (CastTraits::numStages - 1); int32_t next_stage = next % CastTraits::numStages; @@ -1520,10 +1531,6 @@ __global__ void quantize_mxfp8_kernel_cast_only( coords.x = block_coords.x + iter_n * CastTraits::blockIterDim::N; if (next < CastTraits::iterLayout::num && coords.x < cols && coords.y < rows) { if (warpId == 0 && leader) { - if constexpr (CastTraits::_need_wait_group) { - ptx::cp_async_bulk_wait_group_read(); - } - ptx::cp_async_bulk_tensor_2d_global_to_shared( reinterpret_cast(sInput + next_stage * CastTraits::blockIterDim::num), reinterpret_cast(&tensor_map_input), @@ -1984,7 +1991,10 @@ __global__ void quantize_mxfp8_kernel_cast_only( } } - ptx::cp_async_bulk_wait_group_read<0>(); + if (warpId == 0 && leader) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } // NOLINT(readability/fn_size) diff --git a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh index f506b764947..2a481074d27 100644 --- a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh @@ -655,6 +655,15 @@ __global__ void __launch_bounds__(THREADS_NUM) const size_t buff_offset_out = buff * BUFF_OUT_SIZE; const size_t buff_offset_out_t = buff * BUFF_OUT_T_SIZE; + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the master's completion before any thread + // overwrites this reused output buffer. + __syncthreads(); + } + // for stages from 1 to STAGES - 1, we need to update the tensor id // skip updating tensor id if it's the last CTA, and some stages will be out of bounds if (need_update_tensor_id && stage > 0 && (block_offset_Y + stage_offset_Y < rows)) { @@ -673,10 +682,6 @@ __global__ void __launch_bounds__(THREADS_NUM) } if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); - const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -1050,6 +1055,11 @@ __global__ void __launch_bounds__(THREADS_NUM) // } // } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #else NVTE_DEVICE_ERROR("sm_100 or higher is required."); diff --git a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh index 33371de1349..6a76838695c 100644 --- a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh @@ -827,11 +827,16 @@ __global__ void __launch_bounds__(THREADS_NUM) const size_t buff_offset_out = buff * BUFF_OUT_SIZE; const size_t buff_offset_out_t = buff * BUFF_OUT_T_SIZE; - if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the master's completion before any thread + // overwrites this reused output buffer. + __syncthreads(); + } + if (next_stage < STAGES) { const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -1242,6 +1247,11 @@ __global__ void __launch_bounds__(THREADS_NUM) } } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); if constexpr (APPLY_COLUMNWISE_RHT) { destroy_barriers<1>(mbar_rht, is_master_thread); @@ -1413,11 +1423,16 @@ __global__ void __launch_bounds__(THREADS_NUM) const size_t buff_offset_out = buff * BUFF_OUT_SIZE; const size_t buff_offset_out_t = buff * BUFF_OUT_T_SIZE; - if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the master's completion before any thread + // overwrites this reused output buffer. + __syncthreads(); + } + if (next_stage < STAGES) { const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -1804,6 +1819,11 @@ __global__ void __launch_bounds__(THREADS_NUM) } } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } diff --git a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh index 4aa7aa36ee5..3c24f9a4111 100644 --- a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh +++ b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh @@ -81,7 +81,11 @@ constexpr int STAGES = STAGES_Y * STAGES_X; constexpr int BUFFS_NUM = TunableConfig::PREFETCH_STAGES + 1; constexpr int BUFFS_NUM_IN = BUFFS_NUM; constexpr int BUFFS_NUM_OUT = BUFFS_NUM; -constexpr int BUFFS_NUM_OUT_TR = 2; +constexpr int BUFFS_NUM_OUT_TR = TunableConfig::PREFETCH_STAGES + 1; +static_assert(BUFFS_NUM_OUT >= TunableConfig::PREFETCH_STAGES + 1, + "The output buffer ring must be larger than the TMA store wait depth"); +static_assert(BUFFS_NUM_OUT_TR >= TunableConfig::PREFETCH_STAGES + 1, + "The transposed output buffer ring must be larger than the TMA store wait depth"); constexpr int BUFF_DIM_Y = TILE_DIM_Y; constexpr int BUFF_DIM_X = TILE_DIM_X; constexpr int BUFF_SIZE = BUFF_DIM_Y * BUFF_DIM_X; @@ -586,7 +590,12 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D // Wait for TMA transfer to have finished reading shared memory // I.e. the OUT buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read(); + if (leading_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the leading thread's completion to all + // threads before they cooperatively overwrite a reused output buffer. + __syncthreads(); // NVFP4 Quantization rowwise_scaling( @@ -674,6 +683,11 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D } } + if (leading_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + if (leading_thread) { #pragma unroll for (int buff = 0; buff < BUFFS_NUM; ++buff) { diff --git a/transformer_engine/common/fused_rope/fused_mla_kv_rope_mxfp8.cu b/transformer_engine/common/fused_rope/fused_mla_kv_rope_mxfp8.cu new file mode 100644 index 00000000000..2b70ff79837 --- /dev/null +++ b/transformer_engine/common/fused_rope/fused_mla_kv_rope_mxfp8.cu @@ -0,0 +1,451 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +/*! \file fused_mla_kv_rope_mxfp8.cu + * \brief Multi-latent attention key/value split and key RoPE fused with MXFP8 quantization. + * + * MLA's key/value up-projection gives, per head, kv = [k_nope | v], and the key is k_nope + * followed by k_rope, the key's RoPE dimensions that all heads share, with RoPE applied. For + * MXFP8 attention, key and value are quantized row-wise (32 elements of a head) and + * column-wise (32 tokens). Otherwise the key and value are written in BF16 and read again + * by the quantization; this kernel reads kv and k_rope once and writes the MXFP8 key and + * value directly, the same bytes as the MXFP8 quantizer applied to the [s, b * h * d] views + * of the RoPE'd key and of the value. + * + * One CTA covers 32 tokens of two heads of one batch entry. It issues all of its global + * loads up front, stages the tile (RoPE applied) in shared memory, quantizes it with six + * warps row-wise and three warps column-wise, stages the results in shared memory in the + * layout of the CTA's part of the outputs and writes them with 16-byte stores. + */ + +#include +#include +#include + +#include +#include +#include + +#include "../common.h" +#include "../util/logging.h" +#include "../util/ptx.cuh" +#include "../util/ptx_arch_spec.cuh" +#include "../utils.cuh" + +namespace transformer_engine { +namespace { + +// Head dimensions: key without RoPE, RoPE part of the key, value (DeepSeek-V3's MLA). +constexpr int kNopeDim = 128; +constexpr int kRopeDim = 64; +constexpr int kHalfRopeDim = kRopeDim / 2; +constexpr int kValueDim = 128; +constexpr int kKeyDim = kNopeDim + kRopeDim; +constexpr int kKVDim = kNopeDim + kValueDim; + +constexpr int kBlock = 32; // MXFP8 scaling block, and tokens per CTA. +constexpr int kHeadsPerCTA = 2; +constexpr int kRowwiseThreads = 6 * THREADS_PER_WARP; +constexpr int kColwiseThreads = 3 * THREADS_PER_WARP; +constexpr int kThreads = kRowwiseThreads + kColwiseThreads; +constexpr int kPitch = kKeyDim + 8; // BF16 elements per staged token; the padding staggers banks. + +// Global buffers of one MXFP8 output. +struct MXFP8Output { + uint8_t *rowwise_data; + uint8_t *rowwise_scale_inv; + uint8_t *columnwise_data; + uint8_t *columnwise_scale_inv; +}; + +// A quantized tile of a tensor with head dimension D, laid out like its part of the global +// outputs: token r holds the kHeadsPerCTA * D bytes of the CTA's heads. +template +struct StagedOutput { + alignas(16) uint8_t rowwise_data[kBlock * kHeadsPerCTA * D]; + alignas(16) uint8_t columnwise_data[kBlock * kHeadsPerCTA * D]; + alignas(16) uint8_t rowwise_scale_inv[kBlock * kHeadsPerCTA * (D / kBlock)]; + alignas(16) uint8_t columnwise_scale_inv[kHeadsPerCTA * D]; +}; + +constexpr size_t kTileBytes = kHeadsPerCTA * kBlock * kPitch * sizeof(bf16); +constexpr size_t kSharedMemoryBytes = kTileBytes + sizeof(StagedOutput); + +__device__ __forceinline__ bf16 bf16_from_bits(const uint32_t bits) { + return __ushort_as_bfloat16(static_cast(bits & 0xFFFFu)); +} + +__device__ __forceinline__ uint32_t bf16_bits(const bf16 x) { return __bfloat16_as_ushort(x); } + +// Largest magnitude in a BF16 pair of maxima, as FP32 (NaNs are ignored, as by the quantizer). +__device__ __forceinline__ float bf16x2_max(const __nv_bfloat162 m) { + return fmaxf(__bfloat162float(m.x), __bfloat162float(m.y)); +} + +__device__ __forceinline__ __nv_bfloat162 abs_max(const __nv_bfloat162 m, const uint32_t pair) { + return __hmax2(m, __habs2(*reinterpret_cast(&pair))); +} + +// kBlock tokens x W BF16 elements without RoPE, token rows `stride` elements apart, loaded as +// 16-byte vectors before any of them is used. +template +struct PlainLoads { + static constexpr int kVectorsPerToken = W / 8; + static constexpr int kTasks = kBlock * kVectorsPerToken; + static constexpr int kIterations = (kTasks + kThreads - 1) / kThreads; + uint4 vectors[kIterations]; + + __device__ __forceinline__ void load(const bf16 *src, const int64_t stride) { +#pragma unroll + for (int it = 0; it < kIterations; ++it) { + const int task = threadIdx.x + it * kThreads; + if (task < kTasks) { + const int token = task / kVectorsPerToken, vector = task % kVectorsPerToken; + vectors[it] = __ldg(reinterpret_cast(src + token * stride) + vector); + } + } + } + + __device__ __forceinline__ void store(bf16 *tile) const { +#pragma unroll + for (int it = 0; it < kIterations; ++it) { + const int task = threadIdx.x + it * kThreads; + if (task < kTasks) { + const int token = task / kVectorsPerToken, vector = task % kVectorsPerToken; + *reinterpret_cast(tile + token * kPitch + 8 * vector) = vectors[it]; + } + } + } +}; + +// RoPE of k_rope for kBlock tokens, one task per (token, group of 4 rotation pairs). +template +struct RopeLoads { + using CosType = std::conditional_t; + using CosVector = std::conditional_t; // 4 values + static constexpr int kGroups = kHalfRopeDim / 4; + static constexpr int kTasks = kBlock * kGroups; + static_assert(kTasks <= kThreads); + uint4 pairs; // 4 interleaved (x1, x2) pairs + CosVector cos_left, cos_right, sin_left, sin_right; + + __device__ __forceinline__ bool active() const { return threadIdx.x < kTasks; } + + __device__ __forceinline__ void load(const bf16 *k_rope, const int64_t stride, const CosType *cos, + const CosType *sin, const int64_t first_token) { + if (!active()) return; + const int token = threadIdx.x / kGroups, group = threadIdx.x % kGroups; + pairs = __ldg(reinterpret_cast(k_rope + token * stride) + group); + const CosType *c = cos + (first_token + token) * kRopeDim; + const CosType *s = sin + (first_token + token) * kRopeDim; + cos_left = __ldg(reinterpret_cast(c) + group); + cos_right = __ldg(reinterpret_cast(c + kHalfRopeDim) + group); + sin_left = __ldg(reinterpret_cast(s) + group); + sin_right = __ldg(reinterpret_cast(s + kHalfRopeDim) + group); + } + + // Writes x_left = x1 * cos_left - x2 * sin_left and x_right = x2 * cos_right + x1 * sin_right + // to the tile's columns kNopeDim.. (all x_left, then all x_right), with the roundings of + // Megatron-LM's MLA RoPE kernels: in BF16 for BF16 cos / sin (each product and FMA rounded), + // in FP32 rounded once to BF16 for FP32 cos / sin. + __device__ __forceinline__ void store(bf16 *tile) const { + if (!active()) return; + const int token = threadIdx.x / kGroups, group = threadIdx.x % kGroups; + const uint32_t words[4] = {pairs.x, pairs.y, pairs.z, pairs.w}; + uint32_t left[4], right[4]; +#pragma unroll + for (int p = 0; p < 4; ++p) { + const bf16 x1 = bf16_from_bits(words[p]), x2 = bf16_from_bits(words[p] >> 16); + if constexpr (kCosBf16) { + const bf16 *cl = reinterpret_cast(&cos_left); + const bf16 *cr = reinterpret_cast(&cos_right); + const bf16 *sl = reinterpret_cast(&sin_left); + const bf16 *sr = reinterpret_cast(&sin_right); + left[p] = bf16_bits(__hfma(x1, cl[p], __hneg(__hmul(x2, sl[p])))); + right[p] = bf16_bits(__hfma(x1, sr[p], __hmul(x2, cr[p]))); + } else { + const float *cl = reinterpret_cast(&cos_left); + const float *cr = reinterpret_cast(&cos_right); + const float *sl = reinterpret_cast(&sin_left); + const float *sr = reinterpret_cast(&sin_right); + const float f1 = __bfloat162float(x1), f2 = __bfloat162float(x2); + left[p] = bf16_bits(__float2bfloat16_rn(__fmaf_rn(f1, cl[p], -__fmul_rn(f2, sl[p])))); + right[p] = bf16_bits(__float2bfloat16_rn(__fmaf_rn(f1, sr[p], __fmul_rn(f2, cr[p])))); + } + } + bf16 *dst = tile + token * kPitch + kNopeDim + 4 * group; + *reinterpret_cast(dst) = + make_uint2(left[0] | (left[1] << 16), left[2] | (left[3] << 16)); + *reinterpret_cast(dst + kHalfRopeDim) = + make_uint2(right[0] | (right[1] << 16), right[2] | (right[3] << 16)); + } +}; + +// Quantizes head h of the staged tile (kBlock tokens x D) into `out`, as the MXFP8 quantizer +// does: row-wise one (token, 32-element block) per task on the row-wise warps, column-wise two +// adjacent columns per task on the column-wise warps. +template +__device__ __forceinline__ void quantize_head(const bf16 *tile, StagedOutput &out, const int h) { + constexpr int kBlocksPerToken = D / kBlock; + constexpr int kTokenBytes = kHeadsPerCTA * D; + constexpr float kMaxNormRcp = Quantized_Limits::max_norm_rcp; + if (threadIdx.x < kRowwiseThreads) { + for (int task = threadIdx.x; task < kBlock * kBlocksPerToken; task += kRowwiseThreads) { + const int token = task % kBlock, block = task / kBlock; + const uint4 *src = reinterpret_cast(tile + token * kPitch + kBlock * block); + uint32_t words[16]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const uint4 u = src[i]; + words[4 * i] = u.x; + words[4 * i + 1] = u.y; + words[4 * i + 2] = u.z; + words[4 * i + 3] = u.w; + } + __nv_bfloat162 m = __habs2(*reinterpret_cast(&words[0])); +#pragma unroll + for (int i = 1; i < 16; ++i) m = abs_max(m, words[i]); + const e8m0_t biased_exponent = ptx::float_to_e8m0(bf16x2_max(m) * kMaxNormRcp); + const ptx::bf16x2 scale = ptx::exp2f_rcp_2x(biased_exponent); + uint32_t q[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + ptx::mul_cvt_4x(reinterpret_cast(q[i]), + reinterpret_cast(words[2 * i]), scale); + } + uint4 *dst = reinterpret_cast(out.rowwise_data + token * kTokenBytes + h * D + + kBlock * block); + dst[0] = make_uint4(q[0], q[1], q[2], q[3]); + dst[1] = make_uint4(q[4], q[5], q[6], q[7]); + out.rowwise_scale_inv[token * kHeadsPerCTA * kBlocksPerToken + h * kBlocksPerToken + block] = + biased_exponent; + } + } else { + for (int pair = threadIdx.x - kRowwiseThreads; pair < D / 2; pair += kColwiseThreads) { + // Columns 2 * pair and 2 * pair + 1 of all kBlock tokens. + const uint32_t *column = reinterpret_cast(tile) + pair; + uint32_t values[kBlock]; +#pragma unroll + for (int r = 0; r < kBlock; ++r) values[r] = column[r * (kPitch / 2)]; + __nv_bfloat162 m = __habs2(*reinterpret_cast(&values[0])); +#pragma unroll + for (int r = 1; r < kBlock; ++r) m = abs_max(m, values[r]); + const e8m0_t exponent0 = ptx::float_to_e8m0(__bfloat162float(m.x) * kMaxNormRcp); + const e8m0_t exponent1 = ptx::float_to_e8m0(__bfloat162float(m.y) * kMaxNormRcp); + const ptx::bf16x2 scale{ptx::exp2f_rcp(exponent0), ptx::exp2f_rcp(exponent1)}; +#pragma unroll + for (int r = 0; r < kBlock; r += 2) { + uint32_t q; + ptx::mul_cvt_4x(reinterpret_cast(q), + reinterpret_cast(values[r]), scale, + reinterpret_cast(values[r + 1]), scale); + uint8_t *dst = out.columnwise_data + r * kTokenBytes + h * D + 2 * pair; + *reinterpret_cast(dst) = static_cast(q); + *reinterpret_cast(dst + kTokenBytes) = static_cast(q >> 16); + } + *reinterpret_cast(out.columnwise_scale_inv + h * D + 2 * pair) = + static_cast(exponent0 | (exponent1 << 8)); + } + } +} + +// Writes the staged tile. first_row = first_token * num_columns + first_column is the tile's +// first (token, column) row of the [s * b * h, D] outputs; first_colwise_row is the matching +// row of the column-wise scales, [s / 32 * b * h, D]. +template +__device__ __forceinline__ void write_out(const StagedOutput &staged, const int64_t first_row, + const int64_t num_columns, + const int64_t first_colwise_row, const MXFP8Output &out) { + constexpr int kTokenBytes = kHeadsPerCTA * D; + constexpr int kVectorsPerToken = kTokenBytes / 16; + constexpr int kScaleBytesPerToken = kHeadsPerCTA * (D / kBlock); + static_assert(kTokenBytes % 16 == 0 && kScaleBytesPerToken % 4 == 0); + constexpr int kScaleWordsPerToken = kScaleBytesPerToken / 4; + for (int task = threadIdx.x; task < 2 * kBlock * kVectorsPerToken; task += kThreads) { + const bool columnwise = task >= kBlock * kVectorsPerToken; + const int t = task % (kBlock * kVectorsPerToken); + const int token = t / kVectorsPerToken, vector = t % kVectorsPerToken; + const uint8_t *src = + (columnwise ? staged.columnwise_data : staged.rowwise_data) + token * kTokenBytes; + uint8_t *dst = (columnwise ? out.columnwise_data : out.rowwise_data) + + (first_row + token * num_columns) * D; + reinterpret_cast(dst)[vector] = reinterpret_cast(src)[vector]; + } + for (int task = threadIdx.x; task < kBlock * kScaleWordsPerToken; task += kThreads) { + const int token = task / kScaleWordsPerToken, word = task % kScaleWordsPerToken; + uint8_t *dst = out.rowwise_scale_inv + (first_row + token * num_columns) * (D / kBlock); + reinterpret_cast(dst)[word] = reinterpret_cast( + staged.rowwise_scale_inv + token * kScaleBytesPerToken)[word]; + } + for (int task = threadIdx.x; task < kTokenBytes / 16; task += kThreads) { + reinterpret_cast(out.columnwise_scale_inv + first_colwise_row * D)[task] = + reinterpret_cast(staged.columnwise_scale_inv)[task]; + } +} + +template +__global__ void __launch_bounds__(kThreads) + fused_mla_kv_rope_mxfp8_kernel(const bf16 *__restrict__ kv, const bf16 *__restrict__ k_rope, + const int64_t k_rope_stride, + const typename RopeLoads::CosType *__restrict__ cos, + const typename RopeLoads::CosType *__restrict__ sin, + const MXFP8Output key, const MXFP8Output value, + const int num_columns, const int heads) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + extern __shared__ __align__(16) uint8_t shared_memory[]; + bf16 *tiles = reinterpret_cast(shared_memory); // [kHeadsPerCTA][kBlock][kPitch] + auto &key_out = *reinterpret_cast *>(shared_memory + kTileBytes); + auto &value_out = *reinterpret_cast *>(shared_memory + kTileBytes); + + // Column c = batch * heads + head. blockIdx.x: kHeadsPerCTA columns of one batch entry + // (kHeadsPerCTA divides heads), blockIdx.y: kBlock tokens. + const int64_t columns = num_columns; + const int64_t first_column = static_cast(blockIdx.x) * kHeadsPerCTA; + const int64_t token_block = blockIdx.y; + const int64_t first_token = token_block * kBlock; + const int64_t batch_size = columns / heads; + const int64_t batch = first_column / heads; + + PlainLoads k_nope[kHeadsPerCTA]; + PlainLoads v[kHeadsPerCTA]; + RopeLoads rope; +#pragma unroll + for (int h = 0; h < kHeadsPerCTA; ++h) { + const bf16 *src = kv + (first_token * columns + first_column + h) * kKVDim; + k_nope[h].load(src, columns * kKVDim); + v[h].load(src + kNopeDim, columns * kKVDim); + } + rope.load(k_rope + (first_token * batch_size + batch) * k_rope_stride, batch_size * k_rope_stride, + cos, sin, first_token); + + // Key: k_nope and the RoPE'd k_rope, which both heads share. +#pragma unroll + for (int h = 0; h < kHeadsPerCTA; ++h) { + k_nope[h].store(tiles + h * kBlock * kPitch); + rope.store(tiles + h * kBlock * kPitch); + } + __syncthreads(); +#pragma unroll 1 + for (int h = 0; h < kHeadsPerCTA; ++h) { + quantize_head(tiles + h * kBlock * kPitch, key_out, h); + } + __syncthreads(); + write_out(key_out, first_token * columns + first_column, columns, + token_block * columns + first_column, key); + __syncthreads(); + + // Value. +#pragma unroll + for (int h = 0; h < kHeadsPerCTA; ++h) v[h].store(tiles + h * kBlock * kPitch); + __syncthreads(); +#pragma unroll 1 + for (int h = 0; h < kHeadsPerCTA; ++h) { + quantize_head(tiles + h * kBlock * kPitch, value_out, h); + } + __syncthreads(); + write_out(value_out, first_token * columns + first_column, columns, + token_block * columns + first_column, value); +#else + NVTE_DEVICE_THREAD0_ERROR("Fused MLA key/value RoPE with MXFP8 output requires SM 10.0+."); +#endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +} + +bool is_aligned_to(const void *ptr, const size_t alignment) { + return reinterpret_cast(ptr) % alignment == 0; +} + +MXFP8Output check_mxfp8_output(const Tensor &t, const char *name, const size_t rows, + const size_t cols) { + NVTE_CHECK(is_mxfp8_scaling(t.scaling_mode), name, " must be an MXFP8 tensor, got ", + to_string(t.scaling_mode), "."); + NVTE_CHECK(t.has_data() && t.has_columnwise_data(), name, + " needs both row-wise and column-wise data."); + NVTE_CHECK(t.data.dtype == DType::kFloat8E4M3 && t.columnwise_data.dtype == DType::kFloat8E4M3, + name, " must have E4M3 data."); + NVTE_CHECK(!t.with_gemm_swizzled_scales, name, + " must have compact (not GEMM-swizzled) scaling factors."); + const auto [first_dim, last_dim] = t.flat_2d_dims(); + NVTE_CHECK(first_dim == rows && last_dim == cols, name, " must have shape [", rows, ", ", cols, + "], got ", t.shape(), "."); + CheckOutputTensor(t, name); + MXFP8Output out{static_cast(t.data.dptr), static_cast(t.scale_inv.dptr), + static_cast(t.columnwise_data.dptr), + static_cast(t.columnwise_scale_inv.dptr)}; + NVTE_CHECK(is_aligned_to(out.rowwise_data, 16) && is_aligned_to(out.columnwise_data, 16) && + is_aligned_to(out.rowwise_scale_inv, 16) && + is_aligned_to(out.columnwise_scale_inv, 16), + name, " buffers must be 16-byte aligned."); + return out; +} + +} // namespace + +void fused_mla_kv_rope_mxfp8(const Tensor &kv, const Tensor &k_rope, const int k_rope_stride, + const Tensor &cos, const Tensor &sin, Tensor *key, Tensor *value, + cudaStream_t stream) { + NVTE_CHECK(is_supported_by_CC_100(), + "Fused MLA key/value RoPE with MXFP8 output requires SM 10.0+."); + const auto &kv_shape = kv.data.shape; + NVTE_CHECK(kv.data.dtype == DType::kBFloat16, "kv must be BF16."); + NVTE_CHECK(kv_shape.size() == 4 && kv_shape[3] == kKVDim, "kv must have shape [s, b, h, ", kKVDim, + "], got ", kv_shape, "."); + const size_t s = kv_shape[0], b = kv_shape[1], h = kv_shape[2]; + NVTE_CHECK(s % kBlock == 0, "The sequence length (", s, ") must be a multiple of ", kBlock, "."); + NVTE_CHECK(h % kHeadsPerCTA == 0, "The number of heads (", h, ") must be even."); + NVTE_CHECK(s / kBlock <= 65535 && b * h <= static_cast(std::numeric_limits::max()), + "kv shape ", kv_shape, " is too large."); + + NVTE_CHECK(k_rope.data.dtype == DType::kBFloat16, "k_rope must be BF16."); + NVTE_CHECK(k_rope.flat_first_dim() == s * b && k_rope.flat_last_dim() == kRopeDim, + "k_rope must have s * b = ", s * b, " rows of ", kRopeDim, " elements, got shape ", + k_rope.data.shape, "."); + NVTE_CHECK(k_rope_stride >= kRopeDim && k_rope_stride % 8 == 0, "The k_rope row stride (", + k_rope_stride, ") must be at least the row size and a multiple of 8 elements."); + + NVTE_CHECK(cos.data.dtype == sin.data.dtype && + (cos.data.dtype == DType::kBFloat16 || cos.data.dtype == DType::kFloat32), + "cos and sin must both be BF16 or both FP32."); + NVTE_CHECK(cos.flat_last_dim() == kRopeDim && sin.flat_last_dim() == kRopeDim && + cos.flat_first_dim() >= s && sin.flat_first_dim() >= s, + "cos and sin must have at least ", s, " rows of ", kRopeDim, " elements, got shapes ", + cos.data.shape, " and ", sin.data.shape, "."); + + const MXFP8Output key_out = check_mxfp8_output(*key, "key", s, b * h * kKeyDim); + const MXFP8Output value_out = check_mxfp8_output(*value, "value", s, b * h * kValueDim); + NVTE_CHECK(is_aligned_to(kv.data.dptr, 16) && is_aligned_to(k_rope.data.dptr, 16) && + is_aligned_to(cos.data.dptr, 16) && is_aligned_to(sin.data.dptr, 16), + "kv, k_rope, cos and sin must be 16-byte aligned."); + if (s == 0 || b == 0) { + return; + } + + const dim3 grid(b * h / kHeadsPerCTA, s / kBlock); + TRANSFORMER_ENGINE_SWITCH_CONDITION(cos.data.dtype == DType::kBFloat16, kCosBf16, { + using CosType = typename RopeLoads::CosType; + auto kernel = fused_mla_kv_rope_mxfp8_kernel; + NVTE_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + kSharedMemoryBytes)); + kernel<<>>( + static_cast(kv.data.dptr), static_cast(k_rope.data.dptr), + k_rope_stride, static_cast(cos.data.dptr), + static_cast(sin.data.dptr), key_out, value_out, static_cast(b * h), + static_cast(h)); + }); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); +} + +} // namespace transformer_engine + +void nvte_fused_mla_kv_rope_mxfp8(const NVTETensor kv, const NVTETensor k_rope, + const int k_rope_stride, const NVTETensor cos, + const NVTETensor sin, NVTETensor key, NVTETensor value, + cudaStream_t stream) { + NVTE_API_CALL(nvte_fused_mla_kv_rope_mxfp8); + using namespace transformer_engine; + fused_mla_kv_rope_mxfp8(*convertNVTETensorCheck(kv), *convertNVTETensorCheck(k_rope), + k_rope_stride, *convertNVTETensorCheck(cos), *convertNVTETensorCheck(sin), + convertNVTETensorCheck(key), convertNVTETensorCheck(value), stream); +} diff --git a/transformer_engine/common/fused_router/fused_topk_with_score_function.cu b/transformer_engine/common/fused_router/fused_topk_with_score_function.cu index 4ea11f05dce..32d525bb027 100644 --- a/transformer_engine/common/fused_router/fused_topk_with_score_function.cu +++ b/transformer_engine/common/fused_router/fused_topk_with_score_function.cu @@ -265,7 +265,10 @@ __global__ void fused_topk_forward_simple_kernel( // Postprocess: revert bias, softmax, normalization if (expert_bias && (score_function == 0 || score_function == 2)) { for (int i = lane_id; i < topk; i += kThreadsPerWarp) { - topk_scores[i] = topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); + // Recover sigmoid scores directly: subtracting the bias can erase tiny scores. + topk_scores[i] = score_function == 0 + ? intermediate_output[pos_offset + topk_indices[i]] + : topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); } __syncwarp(); } @@ -593,7 +596,10 @@ __global__ void fused_topk_with_score_function_forward_kernel( if constexpr (ScoreFunc == 0 || ScoreFunc == 2) { if (expert_bias) { for (int i = lane_id; i < topk; i += kThreadsPerWarp) { - topk_scores[i] = topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); + // Sigmoid intermediates contain raw scores; sqrtsoftplus intermediates are logits. + topk_scores[i] = + ScoreFunc == 0 ? intermediate_output[pos_offset + topk_indices[i]] + : topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); } __syncwarp(); } diff --git a/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu index f2c0c6b00fe..df19e907fd0 100644 --- a/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu @@ -500,6 +500,8 @@ __launch_bounds__(512, 1) __global__ static void group_row_col_rht_gemm_device_g if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); diff --git a/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu index 11d4620c594..e16d73862c7 100644 --- a/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu @@ -351,6 +351,8 @@ __global__ static void group_rht_gemm_device( if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); using TMEM_LOAD_NEW = cute::SM100::TMEM::LOAD::SM100_TMEM_LOAD_32dp32b64x; diff --git a/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu index e68581f4895..6faaa10ed44 100644 --- a/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu @@ -488,6 +488,8 @@ __launch_bounds__(512, 1) __global__ static void group_row_col_rht_gemm_device( if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); diff --git a/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu index 856046f1f70..df73afeb671 100644 --- a/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu @@ -316,6 +316,8 @@ rht_gemm_device(MShape M, NShape N, KShape K, ClusterTileShape cluster_tile, if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); using TMEM_LOAD_NEW = cute::SM100::TMEM::LOAD::SM100_TMEM_LOAD_32dp32b64x; diff --git a/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu index 4a81a5e966d..94d395526be 100644 --- a/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu @@ -494,6 +494,8 @@ __global__ static void row_col_rht_gemm_device( if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); diff --git a/transformer_engine/common/include/transformer_engine/fused_rope.h b/transformer_engine/common/include/transformer_engine/fused_rope.h index aea5256a2cb..f0e4540b7cc 100644 --- a/transformer_engine/common/include/transformer_engine/fused_rope.h +++ b/transformer_engine/common/include/transformer_engine/fused_rope.h @@ -139,6 +139,47 @@ void nvte_fused_qkv_rope_backward(const NVTETensor q_grad_out, const NVTETensor const int qkv_split_arg_list_0, const int qkv_split_arg_list_1, const int qkv_split_arg_list_2, cudaStream_t stream); +/*! \brief Split multi-latent attention's key/value, apply RoPE to the key and quantize key and + * value to MXFP8. + * + * Each head of kv holds 128 key dimensions without RoPE followed by 128 value dimensions. + * The key is those 128 dimensions followed by the 64 dimensions of k_rope with RoPE + * applied, the same for all heads. For token t, batch entry b and i < 32, with + * (x1, x2) = (k_rope[t, b, 2i], k_rope[t, b, 2i + 1]): + * + * key[t, b, h, 128 + i] = x1 * cos[t, i] - x2 * sin[t, i] + * key[t, b, h, 128 + 32 + i] = x2 * cos[t, 32 + i] + x1 * sin[t, 32 + i] + * + * This is the RoPE of Megatron-LM's MLA kernels (interleaved pairs in, the two halves out), + * with their roundings: with BF16 cos and sin, fma(x1, cos, -(x2 * sin)) and + * fma(x1, sin, x2 * cos) with every operation rounded to BF16; with FP32 cos and sin, the + * same in FP32, rounded once to BF16. + * + * key and value are written as the MXFP8 quantization of their [s, b * h * d] views, row-wise + * (32 elements of a head) and column-wise (32 tokens), with E4M3 data and compact E8M0 scaling + * factors: the bytes the MXFP8 quantizer produces for the BF16 key and value. + * + * Requires SM 10.0+, an even number of heads and a sequence length that is a multiple of 32. + * + * \param[in] kv Key/value, [s, b, h, 256] in BF16. + * \param[in] k_rope Key RoPE dimensions shared by the heads, s * b rows + * (token-major) of 64 BF16 values. + * \param[in] k_rope_stride Elements from one row of k_rope to the next, a multiple + * of 8. + * \param[in] cos Cosines, [max_s, 64] in BF16 or FP32; row t is used for + * token t. + * \param[in] sin Sines, same shape and type as cos. + * \param[in,out] key MXFP8 key of shape [s, b * h * 192], with row-wise and + * column-wise data. + * \param[in,out] value MXFP8 value of shape [s, b * h * 128], with row-wise and + * column-wise data. + * \param[in] stream CUDA stream used for the operation. + */ +void nvte_fused_mla_kv_rope_mxfp8(const NVTETensor kv, const NVTETensor k_rope, + const int k_rope_stride, const NVTETensor cos, + const NVTETensor sin, NVTETensor key, NVTETensor value, + cudaStream_t stream); + #ifdef __cplusplus } // extern "C" #endif diff --git a/transformer_engine/common/normalization/common.h b/transformer_engine/common/normalization/common.h index dd2f3d14591..26fbaf04351 100644 --- a/transformer_engine/common/normalization/common.h +++ b/transformer_engine/common/normalization/common.h @@ -310,6 +310,13 @@ bool use_cudnn_norm_bwd(); bool& use_zero_centered_gamma_in_weight_dtype(); bool use_cudnn_mxfp8_norm_output_in_input_dtype(); +// RMSNorm forward with MXFP8 output by Transformer Engine's fused kernel +// (rmsnorm/rmsnorm_fwd_mxfp8.cu), which also writes GEMM-swizzled scaling factors. +bool is_supported_by_te_rmsnorm_fwd_mxfp8(const Tensor& x, const Tensor& gamma, const Tensor& z); +void rmsnorm_fwd_mxfp8(const Tensor& x, const Tensor& gamma, float epsilon, Tensor* z, + Tensor* rsigma, int multiprocessorCount, bool zero_centered_gamma, + cudaStream_t stream); + } // namespace normalization } // namespace transformer_engine diff --git a/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp b/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp index f56e2ef1fef..c18e6204f80 100644 --- a/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp +++ b/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp @@ -27,11 +27,6 @@ void rmsnorm_fwd(const Tensor &x, const Tensor &gamma, const float epsilon, Tens !is_mxfp8_scaling(z->scaling_mode)) { NVTE_ERROR("Not implemented scaling mode: " + to_string(z->scaling_mode) + "."); } - if (is_mxfp8_scaling(z->scaling_mode)) { - NVTE_CHECK(!z->with_gemm_swizzled_scales, - "MXFP8 output must have scales in compact format, not swizzled for GEMM."); - } - NVTE_CHECK(!x.data.shape.empty(), "x must have at least one dimension."); const auto [rows, cols] = x.flat_2d_dims(); @@ -53,6 +48,26 @@ void rmsnorm_fwd(const Tensor &x, const Tensor &gamma, const float epsilon, Tens CheckOutputTensor(*rsigma, "rsigma"); } + // MXFP8 output: Transformer Engine's fused kernel, which alone writes GEMM-swizzled scales, + // unless cuDNN is requested or the kernel does not support the tensors. + if (is_mxfp8_scaling(z->scaling_mode)) { + const bool te_supported = is_supported_by_te_rmsnorm_fwd_mxfp8(x, gamma, *z); + if (z->with_gemm_swizzled_scales || (!use_cudnn_norm_fwd() && te_supported)) { + NVTE_CHECK(te_supported, + "RMSNorm forward with GEMM-swizzled MXFP8 scales requires FP32, BF16 or FP16 " + "input and weight, and rows and columns that are multiples of 128."); + if (workspace->data.numel() == 0) { + // The fused kernel needs no workspace, but a zero size denotes a workspace query. + workspace->data.shape = {1}; + workspace->data.dtype = DType::kByte; + return; + } + rmsnorm_fwd_mxfp8(x, gamma, epsilon, z, rsigma, multiprocessorCount, zero_centered_gamma, + stream); + return; + } + } + NVTE_Norm_Backend norm_backend; bool is_aligned = true; bool cudnn_backend = use_cudnn_norm_fwd() || is_mxfp8_scaling(z->scaling_mode); diff --git a/transformer_engine/common/normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu b/transformer_engine/common/normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu new file mode 100644 index 00000000000..1ade354ebc2 --- /dev/null +++ b/transformer_engine/common/normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu @@ -0,0 +1,456 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +/*! \file rmsnorm_fwd_mxfp8.cu + * \brief RMSNorm forward fused with MXFP8 quantization. + * + * Normalizes in FP32 and quantizes the FP32 result, row-wise and/or column-wise, with + * the scaling factors written in compact or GEMM-swizzled layout. The output is that of + * the MXFP8 quantizer applied to the FP32 normalized values, which is also what cuDNN's + * fused kernel produces; the two can differ only through the rounding of rsigma. + * + * Two kernels: a pre-pass computes rsigma per row (one warp per row), then one CTA per + * 32x128 tile normalizes the tile, quantizes it and writes its scaling factors. A tile + * only sees 128 columns of a row, so it cannot reduce the row itself; doing so with whole + * rows per CTA would limit the grid to rows / 32 CTAs. When the caller reserves SMs for other + * work, both kernels are launched over chunks of rows that fit on the remaining SMs. + */ + +#include + +#include +#include +#include + +#include "../../cast/mxfp8/swizzle.cuh" +#include "../../common.h" +#include "../../util/cuda_runtime.h" +#include "../../util/ptx_arch_spec.cuh" +#include "../../utils.cuh" +#include "../common.h" + +namespace transformer_engine { +namespace normalization { +namespace { + +constexpr int kScaleBlock = 32; // MXFP8 scaling block size. +constexpr int kTileRows = 32; // One column-wise scaling block. +constexpr int kTileCols = 128; // Four row-wise scaling blocks. +constexpr int kThreads = 128; +constexpr int kWarps = kThreads / THREADS_PER_WARP; +constexpr int kRowsPerWarp = kTileRows / kWarps; +constexpr int kColsPerLane = kTileCols / THREADS_PER_WARP; +constexpr int kLanesPerBlock = kScaleBlock / kColsPerLane; +constexpr int kRsigmaRowsPerCTA = 4; + +static_assert(kRowsPerWarp == 8); +static_assert(kColsPerLane == 4); +static_assert(kLanesPerBlock == 8); + +template +__device__ __forceinline__ void load_elements(T (&dst)[N], const T *src) { + if constexpr (sizeof(T) * N == 16) { + *reinterpret_cast(dst) = *reinterpret_cast(src); + } else if constexpr (sizeof(T) * N == 8) { + *reinterpret_cast(dst) = *reinterpret_cast(src); + } else { +#pragma unroll + for (int i = 0; i < N; ++i) { + dst[i] = src[i]; + } + } +} + +// rsigma = 1 / sqrt(mean(x^2) + epsilon), one warp per row. +template +__global__ void __launch_bounds__(kRsigmaRowsPerCTA *THREADS_PER_WARP) + rmsnorm_mxfp8_rsigma_kernel(const IType *const x, float *const rsigma, const int rows, + const int cols, const float epsilon) { + const int row = blockIdx.x * kRsigmaRowsPerCTA + threadIdx.x / THREADS_PER_WARP; + const int lane = threadIdx.x % THREADS_PER_WARP; + if (row >= rows) { + return; + } + const IType *const x_row = x + static_cast(row) * cols; + float sum = 0.0f; + if constexpr (kAligned) { + constexpr int kVec = 16 / sizeof(IType); + for (int c = lane * kVec; c < cols; c += THREADS_PER_WARP * kVec) { + IType values[kVec]; + load_elements(values, x_row + c); +#pragma unroll + for (int i = 0; i < kVec; ++i) { + const float value = static_cast(values[i]); + sum += value * value; + } + } + } else { + for (int c = lane; c < cols; c += THREADS_PER_WARP) { + const float value = static_cast(x_row[c]); + sum += value * value; + } + } +#pragma unroll + for (int offset = THREADS_PER_WARP / 2; offset > 0; offset /= 2) { + sum += __shfl_xor_sync(0xffffffff, sum, offset); + } + if (lane == 0) { + rsigma[row] = 1.0f / sqrtf(sum / static_cast(cols) + epsilon); + } +} + +// Four FP32 values times per-value multipliers, converted to four FP8 values. +template +__device__ __forceinline__ uint32_t quantize_4x(const float (&values)[kColsPerLane], + const float (&multipliers)[kColsPerLane]) { + using OTypex2 = + std::conditional_t, ptx::fp8e4m3x2, ptx::fp8e5m2x2>; + OTypex2 out[2]; + ptx::mul_cvt_2x(out[0], ptx::floatx2{values[0], values[1]}, + ptx::floatx2{multipliers[0], multipliers[1]}); + ptx::mul_cvt_2x(out[1], ptx::floatx2{values[2], values[3]}, + ptx::floatx2{multipliers[2], multipliers[3]}); + return static_cast(reinterpret_cast(out[0])) | + (static_cast(reinterpret_cast(out[1])) << 16); +} + +struct QuantizeArgs { + void *rowwise_data; + e8m0_t *rowwise_scale_inv; + void *colwise_data; + e8m0_t *colwise_scale_inv; + // Compact layouts: row strides of the scaling-factor tensors. Swizzled layouts: number of + // 4-wide scaling-factor tiles along the blocked dimension. + size_t rowwise_scale_stride; + size_t colwise_scale_stride; +}; + +// One CTA per 32x128 tile: warp w normalizes rows 8w..8w+7, lane l columns 4l..4l+3. +template +__global__ void __launch_bounds__(kThreads) + rmsnorm_mxfp8_quantize_kernel(const IType *const x, const WType *const gamma, + const float *const rsigma, const QuantizeArgs args, + const int cols, const bool zero_centered_gamma, + const bool gamma_in_weight_dtype) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using dispatch::mxfp8::swizzle::gemm_swizzled_scale_idx; + constexpr float kMaxNormRcp = Quantized_Limits::max_norm_rcp; + __shared__ float colwise_amax[kWarps][kTileCols]; + + const int warp = threadIdx.x / THREADS_PER_WARP; + const int lane = threadIdx.x % THREADS_PER_WARP; + const int row_base = blockIdx.x * kTileRows; + const int row0 = row_base + warp * kRowsPerWarp; + const int col0 = blockIdx.y * kTileCols + lane * kColsPerLane; + + // Gamma as applied by the other normalization backends: with a zero-centered gamma, 1 is + // added in FP32, or in the weight type if requested. + float g[kColsPerLane]; + { + WType w[kColsPerLane]; + if constexpr (kAligned) { + load_elements(w, gamma + col0); + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + w[i] = gamma[col0 + i]; + } + } +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + float value = static_cast(w[i]); + if (zero_centered_gamma) { + value = gamma_in_weight_dtype ? static_cast(static_cast(value + 1.0f)) + : value + 1.0f; + } + g[i] = value; + } + } + + float y[kRowsPerWarp][kColsPerLane]; +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + const int row = row0 + r; + const float rs = rsigma[row]; + IType values[kColsPerLane]; + const IType *const src = x + static_cast(row) * cols + col0; + if constexpr (kAligned) { + load_elements(values, src); + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + values[i] = src[i]; + } + } +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + y[r][i] = (static_cast(values[i]) * rs) * g[i]; + } + } + + if constexpr (kRowwise) { + OType *const data = reinterpret_cast(args.rowwise_data); +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + const int row = row0 + r; + // Each 32-column scaling block spans 8 consecutive lanes. + float amax = 0.0f; +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + amax = fmaxf(amax, fabsf(y[r][i])); + } +#pragma unroll + for (int offset = 1; offset < kLanesPerBlock; offset *= 2) { + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, offset)); + } + const e8m0_t biased_exponent = ptx::float_to_e8m0(amax * kMaxNormRcp); + const float multiplier = ptx::exp2f_rcp(biased_exponent); + const float multipliers[kColsPerLane] = {multiplier, multiplier, multiplier, multiplier}; + const uint32_t quantized = quantize_4x(y[r], multipliers); + OType *const dst = data + static_cast(row) * cols + col0; + if constexpr (kAligned) { + *reinterpret_cast(dst) = quantized; + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + reinterpret_cast(dst)[i] = static_cast(quantized >> (8 * i)); + } + } + if (lane % kLanesPerBlock == 0) { + const size_t scale_col = col0 / kScaleBlock; + const size_t idx = kSwizzled + ? gemm_swizzled_scale_idx(row, scale_col, args.rowwise_scale_stride) + : static_cast(row) * args.rowwise_scale_stride + scale_col; + args.rowwise_scale_inv[idx] = biased_exponent; + } + } + } + + if constexpr (kColwise) { + // Each column's scaling block is the tile's 32 rows: reduce over this warp's 8 rows, then + // over the 4 warps through shared memory. +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + float amax = 0.0f; +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + amax = fmaxf(amax, fabsf(y[r][i])); + } + colwise_amax[warp][lane * kColsPerLane + i] = amax; + } + __syncthreads(); + e8m0_t biased_exponents[kColsPerLane]; + float multipliers[kColsPerLane]; +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + float amax = colwise_amax[0][lane * kColsPerLane + i]; +#pragma unroll + for (int w = 1; w < kWarps; ++w) { + amax = fmaxf(amax, colwise_amax[w][lane * kColsPerLane + i]); + } + biased_exponents[i] = ptx::float_to_e8m0(amax * kMaxNormRcp); + multipliers[i] = ptx::exp2f_rcp(biased_exponents[i]); + } + OType *const data = reinterpret_cast(args.colwise_data); +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + const uint32_t quantized = quantize_4x(y[r], multipliers); + OType *const dst = data + static_cast(row0 + r) * cols + col0; + if constexpr (kAligned) { + *reinterpret_cast(dst) = quantized; + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + reinterpret_cast(dst)[i] = static_cast(quantized >> (8 * i)); + } + } + } + if (warp == 0) { + const size_t scale_row = row_base / kScaleBlock; +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + const size_t col = col0 + i; + const size_t idx = kSwizzled + ? gemm_swizzled_scale_idx(col, scale_row, args.colwise_scale_stride) + : scale_row * args.colwise_scale_stride + col; + args.colwise_scale_inv[idx] = biased_exponents[i]; + } + } + } +#else + NVTE_DEVICE_THREAD0_ERROR("RMSNorm forward with MXFP8 output requires SM 10.0+."); +#endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +} + +bool is_aligned_to(const void *ptr, size_t alignment) { + return reinterpret_cast(ptr) % alignment == 0; +} + +// Rows per launch of a kernel with grid rows of grid_cols CTAs, each grid row covering cta_rows +// rows: all rows, unless the caller reserves SMs for other work. Then a launch runs at most as +// many CTAs as fit on the remaining num_sms SMs, in multiples of 128 rows (at least 128). +template +int rows_per_launch(Kernel kernel, const int threads, const int num_sms, const int rows, + const int cta_rows, const int grid_cols) { + if (num_sms <= 0 || num_sms >= cuda::sm_count()) { + return rows; + } + int ctas_per_sm = 0; + NVTE_CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&ctas_per_sm, kernel, threads, 0)); + const int grid_rows = std::max(ctas_per_sm, 1) * num_sms / grid_cols; + return std::min(rows, std::max(grid_rows * cta_rows / 128 * 128, 128)); +} + +template +void launch_rmsnorm_fwd_mxfp8(const Tensor &x, const Tensor &gamma, const float epsilon, Tensor *z, + Tensor *rsigma, const int num_sms, const bool zero_centered_gamma, + const bool gamma_in_weight_dtype, cudaStream_t stream) { + const auto [rows_size_t, cols_size_t] = x.flat_2d_dims(); + const int rows = static_cast(rows_size_t); + const int cols = static_cast(cols_size_t); + const bool rowwise = z->has_data(); + const bool colwise = z->has_columnwise_data(); + const bool swizzled = z->with_gemm_swizzled_scales; + + QuantizeArgs args{}; + if (rowwise) { + args.rowwise_data = z->data.dptr; + args.rowwise_scale_inv = reinterpret_cast(z->scale_inv.dptr); + const size_t scale_cols = z->scale_inv.shape.back(); + args.rowwise_scale_stride = swizzled ? scale_cols / 4 : scale_cols; + } + if (colwise) { + args.colwise_data = z->columnwise_data.dptr; + args.colwise_scale_inv = reinterpret_cast(z->columnwise_scale_inv.dptr); + args.colwise_scale_stride = + swizzled ? z->columnwise_scale_inv.shape.front() / 4 : z->columnwise_scale_inv.shape.back(); + } + + const IType *const x_ptr = reinterpret_cast(x.data.dptr); + const WType *const gamma_ptr = reinterpret_cast(gamma.data.dptr); + float *const rsigma_ptr = reinterpret_cast(rsigma->data.dptr); + + // Vector accesses: 16 bytes of a row of x in the pre-pass, 4 elements of x and gamma and 4 + // bytes of each output in the main kernel. Every row starts aligned since cols % 128 == 0. + const bool aligned = is_aligned_to(x_ptr, 16) && is_aligned_to(gamma_ptr, 4 * sizeof(WType)) && + (!rowwise || is_aligned_to(args.rowwise_data, 4)) && + (!colwise || is_aligned_to(args.colwise_data, 4)); + + TRANSFORMER_ENGINE_SWITCH_CONDITION(aligned, kAligned, { + const auto kernel = rmsnorm_mxfp8_rsigma_kernel; + constexpr int threads = kRsigmaRowsPerCTA * THREADS_PER_WARP; + const int launch_rows = rows_per_launch(kernel, threads, num_sms, rows, kRsigmaRowsPerCTA, 1); + for (int row = 0; row < rows; row += launch_rows) { + const int n = std::min(launch_rows, rows - row); + kernel<<>>( + x_ptr + static_cast(row) * cols, rsigma_ptr + row, n, cols, epsilon); + } + }); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); + + // Arguments for the rows from row on, a multiple of 128. The scaling factors of these rows + // start at a fixed offset in each layout: row-wise, after row rows of scales (swizzled: row / + // 128 rows of 128x4 tiles, the same bytes); column-wise, after row / 32 rows of scales + // (swizzled: row / 128 tiles into each column of tiles). + const auto args_from = [&](const int row) { + QuantizeArgs a = args; + const size_t data_offset = static_cast(row) * cols; + if (rowwise) { + a.rowwise_data = static_cast(args.rowwise_data) + data_offset; + a.rowwise_scale_inv += static_cast(row) * z->scale_inv.shape.back(); + } + if (colwise) { + a.colwise_data = static_cast(args.colwise_data) + data_offset; + a.colwise_scale_inv += + swizzled ? static_cast(row / 128) * 512 + : static_cast(row / kScaleBlock) * z->columnwise_scale_inv.shape.back(); + } + return a; + }; + const auto launch = [&](auto kernel) { + const int launch_rows = + rows_per_launch(kernel, kThreads, num_sms, rows, kTileRows, cols / kTileCols); + for (int row = 0; row < rows; row += launch_rows) { + const int n = std::min(launch_rows, rows - row); + kernel<<>>( + x_ptr + static_cast(row) * cols, gamma_ptr, rsigma_ptr + row, args_from(row), + cols, zero_centered_gamma, gamma_in_weight_dtype); + } + }; + TRANSFORMER_ENGINE_SWITCH_CONDITION( + aligned, kAligned, + TRANSFORMER_ENGINE_SWITCH_CONDITION( + swizzled, kSwizzled, + if (rowwise && colwise) { + launch(rmsnorm_mxfp8_quantize_kernel); + } else if (rowwise) { + launch(rmsnorm_mxfp8_quantize_kernel); + } else { + launch(rmsnorm_mxfp8_quantize_kernel); + });); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); +} + +} // namespace + +bool is_supported_by_te_rmsnorm_fwd_mxfp8(const Tensor &x, const Tensor &gamma, const Tensor &z) { + if (!is_mxfp8_scaling(z.scaling_mode) || !is_supported_by_CC_100()) { + return false; + } + const bool rowwise = z.has_data(); + const bool colwise = z.has_columnwise_data(); + if (!rowwise && !colwise) { + return false; + } + if ((rowwise && z.scale_inv.shape.size() != 2) || + (colwise && z.columnwise_scale_inv.shape.size() != 2)) { + return false; + } + const auto [rows, cols] = x.flat_2d_dims(); + // Full 128x128 tiles, so the scaling factors carry no padding in either layout. + if (rows == 0 || cols == 0 || rows % 128 != 0 || cols % 128 != 0 || + rows > static_cast(std::numeric_limits::max()) || cols / kTileCols > 65535) { + return false; + } + const auto is_supported_input = [](DType t) { + return t == DType::kFloat32 || t == DType::kBFloat16 || t == DType::kFloat16; + }; + if (!is_supported_input(x.data.dtype) || !is_supported_input(gamma.data.dtype)) { + return false; + } + const auto is_supported_output = [](DType t) { + return t == DType::kFloat8E4M3 || t == DType::kFloat8E5M2; + }; + if ((rowwise && !is_supported_output(z.data.dtype)) || + (colwise && !is_supported_output(z.columnwise_data.dtype)) || + (rowwise && colwise && z.data.dtype != z.columnwise_data.dtype)) { + return false; + } + return true; +} + +void rmsnorm_fwd_mxfp8(const Tensor &x, const Tensor &gamma, const float epsilon, Tensor *z, + Tensor *rsigma, const int multiprocessorCount, + const bool zero_centered_gamma, cudaStream_t stream) { + const DType otype = z->has_data() ? z->data.dtype : z->columnwise_data.dtype; + const bool gamma_in_weight_dtype = use_zero_centered_gamma_in_weight_dtype(); + TRANSFORMER_ENGINE_TYPE_SWITCH_FLOAT( + x.data.dtype, IType, + TRANSFORMER_ENGINE_TYPE_SWITCH_FLOAT( + gamma.data.dtype, WType, + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( + otype, OType, + launch_rmsnorm_fwd_mxfp8( + x, gamma, epsilon, z, rsigma, multiprocessorCount, zero_centered_gamma, + gamma_in_weight_dtype, stream);););); // NOLINT(*) +} + +} // namespace normalization +} // namespace transformer_engine diff --git a/transformer_engine/common/recipe/__init__.py b/transformer_engine/common/recipe/__init__.py index 48fa4695ce5..f9abf9de87a 100644 --- a/transformer_engine/common/recipe/__init__.py +++ b/transformer_engine/common/recipe/__init__.py @@ -3,6 +3,7 @@ # See LICENSE for license information. """This module provides predefined FP8 recipes.""" + from __future__ import annotations import abc import os @@ -11,7 +12,6 @@ from dataclasses import field from pydantic.dataclasses import dataclass - _BACKWARD_OVERRIDES = (None, "high_precision", "dequantized") _NVFP4_4OVER6_SCOPES = ("none", "weights", "activations", "all") _NVFP4_4OVER6_ERR_MODES = ("MAE", "MSE") @@ -304,9 +304,6 @@ class Float8CurrentScaling(Recipe): use_power_2_scales: bool = os.getenv("NVTE_FP8_CURRENT_SCALING_POWER_2_SCALES", "0") == "1" fp8_format: Format = Format.HYBRID - fp8_quant_fwd_inp = QParams(power_2_scale=use_power_2_scales, amax_epsilon=0.0) - fp8_quant_fwd_weight = QParams(power_2_scale=use_power_2_scales, amax_epsilon=0.0) - fp8_quant_bwd_grad = QParams(power_2_scale=use_power_2_scales, amax_epsilon=0.0) fp8_gemm_fprop: MMParams = MMParams(use_split_accumulator=False) fp8_gemm_dgrad: MMParams = MMParams(use_split_accumulator=True) fp8_gemm_wgrad: MMParams = MMParams(use_split_accumulator=True) @@ -319,6 +316,14 @@ def __post_init__(self) -> None: assert ( self.backward_override in _BACKWARD_OVERRIDES ), "NVTE_BACKWARD_OVERRIDE must be unset or one of: 'high_precision', 'dequantized'." + self.fp8_quant_fwd_inp = QParams(power_2_scale=self.use_power_2_scales, amax_epsilon=0.0) + self.fp8_quant_fwd_weight = QParams(power_2_scale=self.use_power_2_scales, amax_epsilon=0.0) + self.fp8_quant_bwd_grad = QParams(power_2_scale=self.use_power_2_scales, amax_epsilon=0.0) + + def __setstate__(self, state) -> None: + self.__dict__.update(state) + if "fp8_quant_fwd_inp" not in state: + self.__post_init__() def _make_repr(self) -> str: return ( @@ -432,9 +437,6 @@ class Float8BlockScaling(Recipe): use_f32_scales: bool = os.getenv("NVTE_FP8_BLOCK_SCALING_FP32_SCALES", "0") == "1" fp8_format: Format = Format.E4M3 - fp8_quant_fwd_inp = QParams(power_2_scale=not use_f32_scales, amax_epsilon=0.0) - fp8_quant_fwd_weight = QParams(power_2_scale=not use_f32_scales, amax_epsilon=0.0) - fp8_quant_bwd_grad = QParams(power_2_scale=not use_f32_scales, amax_epsilon=0.0) x_block_scaling_dim: int = 1 w_block_scaling_dim: int = 2 grad_block_scaling_dim: int = 1 @@ -468,6 +470,14 @@ def __post_init__(self) -> None: assert ( self.backward_override in _BACKWARD_OVERRIDES ), "NVTE_BACKWARD_OVERRIDE must be unset or one of: 'high_precision', 'dequantized'." + self.fp8_quant_fwd_inp = QParams(power_2_scale=not self.use_f32_scales, amax_epsilon=0.0) + self.fp8_quant_fwd_weight = QParams(power_2_scale=not self.use_f32_scales, amax_epsilon=0.0) + self.fp8_quant_bwd_grad = QParams(power_2_scale=not self.use_f32_scales, amax_epsilon=0.0) + + def __setstate__(self, state) -> None: + self.__dict__.update(state) + if "fp8_quant_fwd_inp" not in state: + self.__post_init__() def _make_repr(self) -> str: return ( diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index f402f175cf3..19fb8e848ad 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -16,6 +16,7 @@ #include "../common.h" #include "../util/cuda_runtime.h" #include "../util/logging.h" +#include "../util/system.h" #include "transformer_engine/transformer_engine.h" namespace transformer_engine { @@ -24,20 +25,6 @@ namespace { constexpr int MXFP8_BLOCK_SIZE = 32; constexpr int NVFP4_BLOCK_SIZE = 16; -int get_max_dynamic_smem(int device_id = -1) { - static std::vector cache(cuda::num_devices(), -1); - static std::vector flags(cuda::num_devices()); - if (device_id < 0) { - device_id = cuda::current_device(); - } - auto init = [&]() { - NVTE_CHECK_CUDA(cudaDeviceGetAttribute(&cache[device_id], - cudaDevAttrMaxSharedMemoryPerBlockOptin, device_id)); - }; - std::call_once(flags[device_id], init); - return cache[device_id]; -} - constexpr __device__ __host__ int TB_DIM = 32; constexpr __device__ __host__ int NEW_SF_TILE_DIM_K = 16; constexpr __device__ __host__ int N_SF_PER_TD_PER_TILE = 4; @@ -958,6 +945,328 @@ __global__ void __launch_bounds__(TB_DIM* TB_DIM) input, output, M, K, original_M, original_K, flat_bid, grid_dim); } +// MXFP8 scale swizzle with one thread block per scale tile, used when +// NVTE_MXFP8_PER_TILE_SWIZZLE=1. The narrow-K / narrow-M kernels above put 32 tiles in one +// 1024-thread block, so e.g. the scales of a 4096-row activation are swizzled by a single block. +// These kernels give every 128-row tile (row-wise) or 128-column x 32-row block (column-wise) its +// own block, which moves a contiguous chunk with coalesced 16-byte loads and stores through shared +// memory. Multi-tensor launches put several tiles in one block only when the grid would otherwise +// be very large. +constexpr int kRowwiseThreadsPerBlock = 128; +constexpr int kColumnwiseThreadsPerBlock = 256; +constexpr int kMaxTensorsPerLaunch = 64; +// Row-wise: the largest grid before several tiles share a block. +constexpr int kRowwiseMaxBlocks = 1536; +// Column-wise: the largest grid with 32-row chunks before switching to 128-row chunks. +constexpr int kColumnwiseMaxBlocksWith32RowChunks = 1024; + +// Whether MXFP8 swizzles use the per-tile kernels (NVTE_MXFP8_PER_TILE_SWIZZLE=1). +bool use_per_tile_swizzle() { return getenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } + +// The 4 scales packed in `word` (the first one in column first_column), with those in columns +// >= num_columns set to zero. +__device__ __forceinline__ uint32_t zero_padding_columns(uint32_t word, int first_column, + int num_columns) { + const int valid_columns = num_columns - first_column; + if (valid_columns >= 4) return word; + if (valid_columns <= 0) return 0u; + return word & ((1u << (8 * valid_columns)) - 1u); +} + +// numerator / divisor for numerator, divisor < 2^16, with magic = ceil(2^32 / divisor) (exact in +// that range); a divisor of 1 passes magic 0. +__device__ __forceinline__ uint32_t divide_by_magic(uint32_t numerator, uint32_t magic) { + return magic == 0u ? numerator : __umulhi(numerator, magic); +} + +inline uint32_t division_magic(int divisor) { + return divisor <= 1 ? 0u : static_cast(((1ull << 32) + divisor - 1) / divisor); +} + +// Row-wise: tiles [first_tile, first_tile + num_tiles) of one tensor, a contiguous chunk of the +// input and of the output. Shared-memory rows are padded to an odd number of words, so the column +// gathers are free of bank conflicts. Requires num_tiles * 128 * padded_k / 4 < 2^16 (guaranteed +// by the 48 KB shared-memory limit). +__device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_k, + uint32_t words_per_row_magic, int original_m, + int original_k, int first_tile, int num_tiles, + uint32_t* smem) { + const int words_per_row = padded_k >> 2; + const int smem_row_words = words_per_row | 1; + const int num_vectors = + num_tiles * 8 * padded_k; // 16-byte vectors: num_tiles * 128 * padded_k / 16 + const size_t chunk_offset = static_cast(first_tile) * 128 * padded_k; + const int4* input_vectors = reinterpret_cast(input + chunk_offset); + for (int i = threadIdx.x; i < num_vectors; i += kRowwiseThreadsPerBlock) { + const int4 vector = __ldg(input_vectors + i); + const uint32_t words[4] = {static_cast(vector.x), static_cast(vector.y), + static_cast(vector.z), static_cast(vector.w)}; +#pragma unroll + for (int w = 0; w < 4; w++) { + const uint32_t word_index = 4 * i + w; + const uint32_t row = divide_by_magic(word_index, words_per_row_magic); + smem[row * smem_row_words + (word_index - row * words_per_row)] = words[w]; + } + } + __syncthreads(); + + // An output tile holds padded_k / 4 blocks of 512 bytes, i.e. 32 vectors each; vector r of a + // block packs rows r, r + 32, r + 64 and r + 96 of its 4 scale columns. + const int vectors_per_tile = 8 * padded_k; + for (int tile = 0; tile < num_tiles; tile++) { + int4* output_vectors = + reinterpret_cast(output + chunk_offset + static_cast(tile) * 128 * padded_k); + const int tile_first_row = tile * 128; + const int global_first_row = first_tile * 128 + tile_first_row; + for (int v = threadIdx.x; v < vectors_per_tile; v += kRowwiseThreadsPerBlock) { + const int k_block = v >> 5; + const int row_in_group = v & 31; + uint32_t words[4]; +#pragma unroll + for (int group = 0; group < 4; group++) { + const int row = row_in_group + 32 * group; + uint32_t word = smem[(tile_first_row + row) * smem_row_words + k_block]; + if (global_first_row + row >= original_m) word = 0u; + words[group] = zero_padding_columns(word, k_block * 4, original_k); + } + output_vectors[v] = make_int4(static_cast(words[0]), static_cast(words[1]), + static_cast(words[2]), static_cast(words[3])); + } + } +} + +// Column-wise: tile m_tile (128 columns), scale rows [first_k, first_k + kChunkRows) clipped to +// padded_k. Each input row segment is 128 contiguous bytes; each output block of 4 scale rows is +// 512 contiguous bytes. +template +__device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_m, + int padded_k, int original_m, + int original_k, int m_tile, int first_k, + int4* smem) { + const int num_rows = min(kChunkRows, padded_k - first_k); // multiple of 4 + const int num_vectors = num_rows * 8; + for (int i = threadIdx.x; i < num_vectors; i += kColumnwiseThreadsPerBlock) { + const int row = i >> 3; + const int vector_in_row = i & 7; + smem[i] = __ldg(reinterpret_cast( + input + static_cast(first_k + row) * padded_m + m_tile * 128) + + vector_in_row); + } + __syncthreads(); + + const uint8_t* smem_bytes = reinterpret_cast(smem); + const size_t first_output_block = static_cast(m_tile) * (padded_k >> 2) + (first_k >> 2); + for (int v = threadIdx.x; v < num_vectors; v += kColumnwiseThreadsPerBlock) { + const int k_block = v >> 5; + const int column_in_group = v & 31; + uint32_t words[4]; +#pragma unroll + for (int group = 0; group < 4; group++) { + const int column = column_in_group + 32 * group; + uint32_t word = 0u; + if (m_tile * 128 + column < original_m) { +#pragma unroll + for (int j = 0; j < 4; j++) { + const int row = k_block * 4 + j; + if (first_k + row < original_k) { + word |= static_cast(smem_bytes[row * 128 + column]) << (8 * j); + } + } + } + words[group] = word; + } + reinterpret_cast(output + (first_output_block + k_block) * 512)[column_in_group] = + make_int4(static_cast(words[0]), static_cast(words[1]), + static_cast(words[2]), static_cast(words[3])); + } +} + +__global__ void __launch_bounds__(kRowwiseThreadsPerBlock) + swizzle_rowwise_per_tile_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, + int padded_k, uint32_t words_per_row_magic, int original_m, + int original_k) { + extern __shared__ uint32_t rowwise_smem[]; + swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, original_k, + blockIdx.x, 1, rowwise_smem); +} + +__global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) + swizzle_columnwise_per_tile_kernel(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_m, int padded_k, + int original_m, int original_k) { + __shared__ int4 columnwise_smem[32 * 8]; + swizzle_columnwise_chunk<32>(input, output, padded_m, padded_k, original_m, original_k, + blockIdx.x, blockIdx.y * 32, columnwise_smem); +} + +// The scales of one tensor: padded_m (a multiple of 128) by padded_k (a multiple of 4) in +// row-wise terms; for column-wise scales, m runs along the columns of the data. Scales at +// m >= original_m or k >= original_k are padding and written as zeros. +struct PerTileSwizzleTensor { + const uint8_t* input; + uint8_t* output; + int padded_m, padded_k, original_m, original_k; +}; + +struct PerTileMultiSwizzleArgs { + PerTileSwizzleTensor tensors[kMaxTensorsPerLaunch]; + int first_block[kMaxTensorsPerLaunch + 1]; + uint32_t words_per_row_magic[kMaxTensorsPerLaunch]; // row-wise only + int tiles_per_block; // row-wise only + int num_tensors; +}; + +__device__ __forceinline__ int tensor_of_block(const PerTileMultiSwizzleArgs& args, int block) { + int tensor = 0; + while (tensor + 1 < args.num_tensors && args.first_block[tensor + 1] <= block) tensor++; + return tensor; +} + +__global__ void __launch_bounds__(kRowwiseThreadsPerBlock) + multi_tensor_swizzle_rowwise_per_tile_kernel(const PerTileMultiSwizzleArgs args) { + extern __shared__ uint32_t rowwise_multi_smem[]; + const int t = tensor_of_block(args, blockIdx.x); + const int tiles_per_block = args.tiles_per_block; + const int first_tile = (blockIdx.x - args.first_block[t]) * tiles_per_block; + const int num_tiles = min(tiles_per_block, args.tensors[t].padded_m / 128 - first_tile); + swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_k, + args.words_per_row_magic[t], args.tensors[t].original_m, + args.tensors[t].original_k, first_tile, num_tiles, rowwise_multi_smem); +} + +template +__global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) + multi_tensor_swizzle_columnwise_per_tile_kernel(const PerTileMultiSwizzleArgs args) { + __shared__ int4 columnwise_multi_smem[kChunkRows * 8]; + const int t = tensor_of_block(args, blockIdx.x); + const int block_in_tensor = blockIdx.x - args.first_block[t]; + const int num_m_tiles = args.tensors[t].padded_m / 128; + const int m_tile = block_in_tensor % num_m_tiles; + const int first_k = (block_in_tensor / num_m_tiles) * kChunkRows; + swizzle_columnwise_chunk(args.tensors[t].input, args.tensors[t].output, + args.tensors[t].padded_m, args.tensors[t].padded_k, + args.tensors[t].original_m, args.tensors[t].original_k, + m_tile, first_k, columnwise_multi_smem); +} + +inline bool is_16_byte_aligned(const void* ptr) { + return (reinterpret_cast(ptr) & 15u) == 0; +} + +inline bool per_tile_swizzle_supported(const PerTileSwizzleTensor& tensor, bool rowwise) { + if (tensor.padded_m <= 0 || tensor.padded_k <= 0 || tensor.padded_m % 128 != 0 || + tensor.padded_k % 4 != 0) { + return false; + } + if (!is_16_byte_aligned(tensor.input) || !is_16_byte_aligned(tensor.output)) return false; + if (rowwise && 128 * ((tensor.padded_k >> 2) | 1) * 4 > 48 * 1024) return false; + return true; +} + +// One kernel for up to kMaxTensorsPerLaunch tensors, all row-wise or all column-wise. Row-wise: +// the tiles per block are the smallest power of two that keeps the grid within kRowwiseMaxBlocks +// (shared memory permitting). Column-wise: 128-row chunks once 32-row chunks would exceed +// kColumnwiseMaxBlocksWith32RowChunks blocks. Returns false, without launching anything, if any +// tensor is not supported. +inline bool launch_multi_tensor_per_tile_swizzle(const PerTileSwizzleTensor* tensors, + int num_tensors, bool rowwise, + cudaStream_t stream) { + if (num_tensors <= 0 || num_tensors > kMaxTensorsPerLaunch) return false; + PerTileMultiSwizzleArgs args; + args.num_tensors = num_tensors; + args.first_block[0] = 0; + int64_t total_tiles = 0; + int max_smem_row_words = 1; + for (int i = 0; i < num_tensors; i++) { + if (!per_tile_swizzle_supported(tensors[i], rowwise)) return false; + args.tensors[i] = tensors[i]; + args.words_per_row_magic[i] = division_magic(tensors[i].padded_k >> 2); + if (rowwise) { + total_tiles += tensors[i].padded_m / 128; + max_smem_row_words = max(max_smem_row_words, (tensors[i].padded_k >> 2) | 1); + } else { + total_tiles += + static_cast(tensors[i].padded_m / 128) * ((tensors[i].padded_k + 31) / 32); + } + } + int tiles_per_block = 1, chunk_rows = 32; + if (rowwise) { + const int max_tiles_per_block = (48 * 1024) / (128 * max_smem_row_words * 4); + while (total_tiles > static_cast(kRowwiseMaxBlocks) * tiles_per_block && + tiles_per_block * 2 <= max_tiles_per_block) { + tiles_per_block *= 2; + } + } else { + if (total_tiles > kColumnwiseMaxBlocksWith32RowChunks) chunk_rows = 128; + } + args.tiles_per_block = tiles_per_block; + for (int i = 0; i < num_tensors; i++) { + const PerTileSwizzleTensor& tensor = tensors[i]; + const int blocks = + rowwise ? (tensor.padded_m / 128 + tiles_per_block - 1) / tiles_per_block + : (tensor.padded_m / 128) * ((tensor.padded_k + chunk_rows - 1) / chunk_rows); + args.first_block[i + 1] = args.first_block[i] + blocks; + } + for (int i = num_tensors + 1; i <= kMaxTensorsPerLaunch; i++) { + args.first_block[i] = args.first_block[num_tensors]; + } + const int grid = args.first_block[num_tensors]; + if (grid == 0) return true; + if (rowwise) { + multi_tensor_swizzle_rowwise_per_tile_kernel<<< + grid, kRowwiseThreadsPerBlock, tiles_per_block * 128 * max_smem_row_words * 4, stream>>>( + args); + } else if (chunk_rows == 128) { + multi_tensor_swizzle_columnwise_per_tile_kernel<128> + <<>>(args); + } else { + multi_tensor_swizzle_columnwise_per_tile_kernel<32> + <<>>(args); + } + NVTE_CHECK_CUDA(cudaGetLastError()); + return true; +} + +inline bool launch_rowwise_per_tile_swizzle(const PerTileSwizzleTensor& tensor, + cudaStream_t stream) { + if (!per_tile_swizzle_supported(tensor, true)) return false; + swizzle_rowwise_per_tile_kernel<<> 2) | 1) * 4, stream>>>( + tensor.input, tensor.output, tensor.padded_k, division_magic(tensor.padded_k >> 2), + tensor.original_m, tensor.original_k); + NVTE_CHECK_CUDA(cudaGetLastError()); + return true; +} + +inline bool launch_columnwise_per_tile_swizzle(const PerTileSwizzleTensor& tensor, + cudaStream_t stream) { + if (!per_tile_swizzle_supported(tensor, false)) return false; + dim3 grid(tensor.padded_m / 128, (tensor.padded_k + 31) / 32); + swizzle_columnwise_per_tile_kernel<<>>( + tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, + tensor.original_k); + NVTE_CHECK_CUDA(cudaGetLastError()); + return true; +} + +// A single tensor uses the one-tile-per-block kernels unless its grid would be large; several +// tensors go through one multi-tensor launch. +inline bool launch_per_tile_swizzle(const PerTileSwizzleTensor* tensors, int num_tensors, + bool rowwise, cudaStream_t stream) { + const bool grid_too_large = + num_tensors == 1 && (rowwise ? tensors[0].padded_m / 128 > kRowwiseMaxBlocks + : static_cast(tensors[0].padded_m / 128) * + ((tensors[0].padded_k + 31) / 32) > + kColumnwiseMaxBlocksWith32RowChunks); + if (num_tensors == 1 && !grid_too_large) { + return rowwise ? launch_rowwise_per_tile_swizzle(tensors[0], stream) + : launch_columnwise_per_tile_swizzle(tensors[0], stream); + } + return launch_multi_tensor_per_tile_swizzle(tensors, num_tensors, rowwise, stream); +} + } // namespace void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t stream) { @@ -1113,7 +1422,17 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const int narrow_k_slm_size = TB_DIM * num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - if (num_tiles_k < TB_DIM && narrow_k_slm_size <= get_max_dynamic_smem()) { + const PerTileSwizzleTensor per_tile_tensor{static_cast(input_scale_inv_ptr), + static_cast(output_scale_inv_ptr), + m, + k, + original_M, + original_K}; + if (scaling_mode == NVTE_MXFP8_1D_SCALING && use_per_tile_swizzle() && + per_tile_swizzle_supported(per_tile_tensor, true)) { + launch_per_tile_swizzle(&per_tile_tensor, 1, true, stream); + } else if (num_tiles_k < TB_DIM && static_cast(narrow_k_slm_size) <= + cuda::max_shared_memory_per_block_optin()) { // Narrow-K: batch TB_DIM M-tiles per block, fully utilizing all threads. dim3 num_blocks_narrow(DIVUP(num_tiles_m, TB_DIM)); NVTE_CHECK_CUDA( @@ -1170,7 +1489,17 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const int narrow_m_slm_size = TB_DIM * num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - if (num_tiles_m < TB_DIM && narrow_m_slm_size <= get_max_dynamic_smem()) { + const PerTileSwizzleTensor per_tile_tensor{ + static_cast(input->columnwise_scale_inv.dptr), + static_cast(output->columnwise_scale_inv.dptr), + m, + k, + original_M, + original_K}; + if (use_per_tile_swizzle() && per_tile_swizzle_supported(per_tile_tensor, false)) { + launch_per_tile_swizzle(&per_tile_tensor, 1, false, stream); + } else if (num_tiles_m < TB_DIM && static_cast(narrow_m_slm_size) <= + cuda::max_shared_memory_per_block_optin()) { // Narrow-M: batch TB_DIM K-tiles per block, fully utilizing all threads. dim3 num_blocks_narrow(DIVUP(num_tiles_k, TB_DIM)); NVTE_CHECK_CUDA( @@ -1447,6 +1776,42 @@ void launch_multi_tensor_unswizzle_scaling_factors(MultiSwizzleArgs& kernel_args } } +namespace { + +// MXFP8 multi-tensor swizzle with the per-tile kernels. Returns false without launching anything +// if any tensor is not supported (the caller then uses the generic kernels, which also report +// shape errors). +bool multi_tensor_per_tile_swizzle(const std::vector& input, std::vector& output, + bool rowwise, cudaStream_t stream) { + const size_t num_tensors = input.size(); + std::vector tensors(num_tensors); + for (size_t i = 0; i < num_tensors; i++) { + if (!is_mxfp8_scaling(input[i]->scaling_mode)) return false; + const auto& in_scales = rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; + const auto& out_scales = rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; + if (in_scales.shape.size() != 2) return false; + const int padded_m = static_cast(rowwise ? in_scales.shape[0] : in_scales.shape[1]); + const int padded_k = static_cast(rowwise ? in_scales.shape[1] : in_scales.shape[0]); + if (padded_m <= 0 || padded_k <= 0 || padded_m % 128 != 0 || padded_k % 4 != 0) return false; + if (out_scales.numel() != static_cast(padded_m) * padded_k) return false; + const auto [first_dim, last_dim] = input[i]->flat_2d_dims(); + tensors[i] = {static_cast(in_scales.dptr), + static_cast(out_scales.dptr), + padded_m, + padded_k, + static_cast(rowwise ? first_dim : last_dim), + static_cast((rowwise ? last_dim : first_dim) / MXFP8_BLOCK_SIZE)}; + if (!per_tile_swizzle_supported(tensors[i], rowwise)) return false; + } + for (size_t first = 0; first < num_tensors; first += kMaxTensorsPerLaunch) { + const int count = static_cast(std::min(kMaxTensorsPerLaunch, num_tensors - first)); + launch_per_tile_swizzle(tensors.data() + first, count, rowwise, stream); + } + return true; +} + +} // namespace + void multi_tensor_swizzle_scaling_factors(const std::vector& input, std::vector& output, cudaStream_t stream, bool check_scale_inv_shapes) { @@ -1482,6 +1847,10 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const bool rowwise_swizzle = all_has_data || all_nvfp4; const bool columnwise_swizzle = all_has_columnwise_data && !all_nvfp4; + if (!all_nvfp4 && use_per_tile_swizzle() && + multi_tensor_per_tile_swizzle(input, output, rowwise_swizzle, stream)) { + return; + } constexpr int SF_TILE_DIM_M = 128; constexpr int SF_TILE_DIM_K = 4; @@ -1538,7 +1907,8 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const int narrow_k_slm = TB_DIM * num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); all_narrow_k = - all_narrow_k && (num_tiles_k < TB_DIM) && (narrow_k_slm <= get_max_dynamic_smem()); + all_narrow_k && (num_tiles_k < TB_DIM) && + (static_cast(narrow_k_slm) <= cuda::max_shared_memory_per_block_optin()); int vec_load_size_i = (num_tiles_k - 1) % 4 + 1; // We use the minimum vec_load_size across all tensors. // TODO(zhongbo): fix vec_load_size for NVFP4 @@ -1610,7 +1980,8 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const int narrow_m_slm = TB_DIM * num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); all_narrow_m = - all_narrow_m && (num_tiles_m < TB_DIM) && (narrow_m_slm <= get_max_dynamic_smem()); + all_narrow_m && (num_tiles_m < TB_DIM) && + (static_cast(narrow_m_slm) <= cuda::max_shared_memory_per_block_optin()); int vec_load_size_i = (num_tiles_k - 1) % 4 + 1; // We use the minimum vec_load_size across all tensors. vec_load_size = std::min(vec_load_size, vec_load_size_i); diff --git a/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu b/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu index 02d64bcfffa..2d1a62e0fae 100644 --- a/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu +++ b/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu @@ -230,8 +230,8 @@ __global__ void __launch_bounds__(THREADS_PER_BLOCK) // Wait for TMA transfer to have finished reading shared memory. // Create a "bulk async-group" out of the previous bulk copy operation. ptx::cp_async_bulk_commit_group(); - // Wait for the group to have completed reading from shared memory. - ptx::cp_async_bulk_wait_group_read<0>(); + // Wait for the group to have completed the shared-to-global transfer. + ptx::cp_async_bulk_wait_group(); } #else // Step 4 Alternative (when TMA is not available, skip writing to shared memory) diff --git a/transformer_engine/common/triton/cross_entropy.py b/transformer_engine/common/triton/cross_entropy.py index 47c9a924b31..3d1fd531782 100644 --- a/transformer_engine/common/triton/cross_entropy.py +++ b/transformer_engine/common/triton/cross_entropy.py @@ -8,6 +8,41 @@ import triton.language as tl +@triton.jit +def _is_nan(x): + """NaN test that works on Triton tensors and scalars: NaN is the only value unequal to itself.""" + return x != x # pylint: disable=comparison-with-itself + + +@triton.jit +def _fold_block_argmax(x, i, offsets, n_cols, run_max, run_idx, run_nan): + """Fold one block of a row into a running argmax with torch.argmax's semantics: NaN counts + as the maximum, and the first maximum (or the first NaN) wins. + + tl.max(..., return_indices=True) is not NaN-safe (its comparisons are false for NaN, so a + block with a NaN can give a pair that is neither the maximum nor the NaN). NaN is therefore + found separately and the maximum is taken over the other values. run_max is kept apart from + the loss's running maximum, which is unchanged.""" + is_nan = _is_nan(x) + block_has_nan = tl.max(is_nan.to(tl.int32)) > 0 + first_nan = tl.min(tl.where(is_nan, offsets, n_cols)) + block_max, block_idx = tl.max( + tl.where(is_nan, float("-inf"), x), + axis=0, + return_indices=True, + return_indices_tie_break_left=True, + ) + if run_nan == 0: + if block_has_nan: + run_idx = first_nan + run_nan = 1 + elif block_max > run_max: + # Only a strictly larger maximum replaces an earlier one: first index wins. + run_idx = i + block_idx + run_max = block_max + return run_max, run_idx, run_nan + + @triton.jit def cross_entropy_forward_kernel( X_ptr, @@ -18,6 +53,7 @@ def cross_entropy_forward_kernel( Y_ptr, loss_ptr, stats_ptr, + argmax_ptr, n_non_ignore, n_cols, n_rows_1, @@ -25,9 +61,11 @@ def cross_entropy_forward_kernel( label_smoothing: tl.constexpr, COUNT_NON_IGNORE: tl.constexpr, COPY_INPUT: tl.constexpr, + COMPUTE_ARGMAX: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): - """Compute single-rank loss/statistics and optionally preserve the input.""" + """Compute single-rank loss/statistics and optionally preserve the input and find the + argmax of each row.""" row = tl.program_id(0).to(tl.int64) row_0 = row // n_rows_1 @@ -43,6 +81,9 @@ def cross_entropy_forward_kernel( m = float("-inf") d = 0.0 x_sum = 0.0 + argmax = 0 + argmax_max = float("-inf") + argmax_nan = 0 for i in range(0, n_cols, BLOCK_SIZE): offsets = i + tl.arange(0, BLOCK_SIZE) mask = offsets < n_cols @@ -50,6 +91,10 @@ def cross_entropy_forward_kernel( if COPY_INPUT: tl.store(saved_input_ptr + offsets, x, mask=mask) x = x.to(tl.float32) + if COMPUTE_ARGMAX: + argmax_max, argmax, argmax_nan = _fold_block_argmax( + x, i, offsets, n_cols, argmax_max, argmax, argmax_nan + ) block_max = tl.max(x) m_new = tl.maximum(m, block_max) d = d * tl.exp(m - m_new) + tl.sum(tl.exp(x - m_new)) @@ -59,6 +104,8 @@ def cross_entropy_forward_kernel( tl.store(stats_ptr + row * 2, m) tl.store(stats_ptr + row * 2 + 1, d) + if COMPUTE_ARGMAX: + tl.store(argmax_ptr + row, argmax.to(tl.int64)) if y == ignore_idx: tl.store(loss_ptr + row, 0.0) @@ -94,6 +141,8 @@ def cross_entropy_tp_pre_kernel( COUNT_NON_IGNORE: tl.constexpr, COMPUTE_X_SUM: tl.constexpr, COPY_INPUT: tl.constexpr, + COMPUTE_ARGMAX: tl.constexpr, + LOCAL_DATA_SIZE: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """Compute the local statistics needed by tensor-parallel cross entropy.""" @@ -118,6 +167,9 @@ def cross_entropy_tp_pre_kernel( m = float("-inf") d = 0.0 x_sum = 0.0 + argmax = 0 + argmax_max = float("-inf") + argmax_nan = 0 for i in range(0, n_cols, BLOCK_SIZE): offsets = i + tl.arange(0, BLOCK_SIZE) mask = offsets < n_cols @@ -125,6 +177,10 @@ def cross_entropy_tp_pre_kernel( if COPY_INPUT: tl.store(saved_input_ptr + offsets, x, mask=mask) x = x.to(tl.float32) + if COMPUTE_ARGMAX: + argmax_max, argmax, argmax_nan = _fold_block_argmax( + x, i, offsets, n_cols, argmax_max, argmax, argmax_nan + ) block_max = tl.max(x) m_new = tl.maximum(m, block_max) d = d * tl.exp(m - m_new) + tl.sum(tl.exp(x - m_new)) @@ -132,12 +188,17 @@ def cross_entropy_tp_pre_kernel( if COMPUTE_X_SUM: x_sum += tl.sum(tl.where(mask, x, 0.0)) - local_data_ptr += row * 4 + local_data_ptr += row * LOCAL_DATA_SIZE tl.store(local_data_ptr, m) tl.store(local_data_ptr + 1, d) tl.store(local_data_ptr + 2, x_y) if COMPUTE_X_SUM: tl.store(local_data_ptr + 3, x_sum) + if COMPUTE_ARGMAX: + # The local index travels with the FP32 statistics as raw bits, followed by the local + # maximum it belongs to, NaN if the shard holds a NaN. + tl.store(local_data_ptr + 4, argmax.to(tl.float32, bitcast=True)) + tl.store(local_data_ptr + 5, tl.where(argmax_nan != 0, float("nan"), argmax_max)) @triton.jit @@ -146,27 +207,44 @@ def cross_entropy_tp_post_kernel( Y_ptr, loss_ptr, stats_ptr, + argmax_ptr, world_size, n_rows, n_cols, ignore_idx, label_smoothing: tl.constexpr, + COMPUTE_ARGMAX: tl.constexpr, + LOCAL_DATA_SIZE: tl.constexpr, ): """Combine tensor-parallel statistics and compute loss/global statistics.""" row = tl.program_id(0).to(tl.int64) - data_ptr = gathered_data_ptr + row * 4 + data_ptr = gathered_data_ptr + row * LOCAL_DATA_SIZE m = tl.load(data_ptr) d = tl.load(data_ptr + 1) x_y = tl.load(data_ptr + 2) x_sum = 0.0 if label_smoothing > 0: x_sum = tl.load(data_ptr + 3) + argmax = 0 + argmax_max = 0.0 + if COMPUTE_ARGMAX: + argmax = tl.load(data_ptr + 4).to(tl.int32, bitcast=True).to(tl.int64) + argmax_max = tl.load(data_ptr + 5) for rank_idx in range(1, world_size): - rank_data_ptr = data_ptr + rank_idx * n_rows * 4 + rank_data_ptr = data_ptr + rank_idx * n_rows * LOCAL_DATA_SIZE m_new = tl.load(rank_data_ptr) d_new = tl.load(rank_data_ptr + 1) + if COMPUTE_ARGMAX: + # Ranks hold consecutive vocabulary shards, so the first rank with the largest + # maximum (or with a NaN, which counts as the maximum) holds the global argmax. + rank_argmax_max = tl.load(rank_data_ptr + 5) + if not _is_nan(argmax_max): + if _is_nan(rank_argmax_max) or (rank_argmax_max > argmax_max): + local_argmax = tl.load(rank_data_ptr + 4).to(tl.int32, bitcast=True) + argmax = rank_idx.to(tl.int64) * n_cols + local_argmax.to(tl.int64) + argmax_max = rank_argmax_max global_m = tl.maximum(m, m_new) d = d * tl.exp(m - global_m) + d_new * tl.exp(m_new - global_m) m = global_m @@ -176,6 +254,8 @@ def cross_entropy_tp_post_kernel( tl.store(stats_ptr + row * 2, m) tl.store(stats_ptr + row * 2 + 1, d) + if COMPUTE_ARGMAX: + tl.store(argmax_ptr + row, argmax) y = tl.load(Y_ptr + row) if y == ignore_idx: diff --git a/transformer_engine/common/util/cuda_runtime.cpp b/transformer_engine/common/util/cuda_runtime.cpp index d23f11ccc6d..17e5026bf09 100644 --- a/transformer_engine/common/util/cuda_runtime.cpp +++ b/transformer_engine/common/util/cuda_runtime.cpp @@ -10,7 +10,9 @@ #include #include +#include #include +#include #include "../common.h" #include "../util/cuda_driver.h" @@ -143,6 +145,37 @@ int sm_count(int device_id) { return cache[device_id]; } +size_t max_shared_memory_per_block_optin(int device_id) { + static std::vector cache(num_devices(), 0); + static std::vector flags(num_devices()); + if (device_id < 0) { + device_id = current_device(); + } + NVTE_CHECK(0 <= device_id && device_id < num_devices(), "invalid CUDA device ID"); + auto init = [&]() { + int value; + NVTE_CHECK_CUDA( + cudaDeviceGetAttribute(&value, cudaDevAttrMaxSharedMemoryPerBlockOptin, device_id)); + cache[device_id] = static_cast(value); + }; + std::call_once(flags[device_id], init); + return cache[device_id]; +} + +size_t static_shared_memory_size(const void *kernel) { + static std::map, size_t> cache; + static std::mutex mutex; + const auto key = std::make_pair(kernel, current_device()); + const std::lock_guard lock(mutex); + auto it = cache.find(key); + if (it == cache.end()) { + cudaFuncAttributes attr; + NVTE_CHECK_CUDA(cudaFuncGetAttributes(&attr, kernel)); + it = cache.emplace(key, attr.sharedSizeBytes).first; + } + return it->second; +} + void stream_priority_range(int *low_priority, int *high_priority, int device_id) { static std::vector> cache(num_devices()); static std::vector flags(num_devices()); diff --git a/transformer_engine/common/util/cuda_runtime.h b/transformer_engine/common/util/cuda_runtime.h index 0f355940010..02f95c08f1e 100644 --- a/transformer_engine/common/util/cuda_runtime.h +++ b/transformer_engine/common/util/cuda_runtime.h @@ -38,6 +38,30 @@ int sm_arch(int device_id = -1); */ int sm_count(int device_id = -1); +/* \brief Maximum shared memory per block available with opt-in + * + * This is the upper bound for the static plus dynamic shared memory + * of a single thread block, after raising the kernel's + * cudaFuncAttributeMaxDynamicSharedMemorySize attribute. + * + * \param[in] device_id CUDA device (default is current device) + * + * \return Shared memory size in bytes + */ +size_t max_shared_memory_per_block_optin(int device_id = -1); + +/* \brief Static shared memory used by a compiled kernel on the current device + * + * Size of the kernel's fixed-size __shared__ arrays, including those + * declared in device functions it calls, as reported by + * cudaFuncGetAttributes. The result is cached per kernel and device. + * + * \param[in] kernel Pointer to the __global__ function + * + * \return Shared memory size in bytes + */ +size_t static_shared_memory_size(const void *kernel); + /* \brief Minimum and maximum stream priorities supported on device * * \param[in] device_id CUDA device (default is current device) diff --git a/transformer_engine/common/util/ptx.cuh b/transformer_engine/common/util/ptx.cuh index 49686437c63..74fd2261f58 100644 --- a/transformer_engine/common/util/ptx.cuh +++ b/transformer_engine/common/util/ptx.cuh @@ -198,7 +198,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx_cta_relaxed_shared_cta __device__ __forceinline__ void fence_mbarrier_init_release_cluster() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm volatile("fence.mbarrier_init.release.cluster;"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); #else NVTE_DEVICE_ERROR("fence_mbarrier_init_release_cluster is only supported on SM 10.0+."); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) @@ -391,7 +391,7 @@ __device__ __forceinline__ void cp_async_bulk_tensor_2d_shared_to_global( // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group __device__ __forceinline__ void cp_async_bulk_wait_group() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group 0;"); + asm volatile("cp.async.bulk.wait_group 0;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -401,7 +401,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group() { template __device__ __forceinline__ void cp_async_bulk_wait_group_read() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 0;"); + asm volatile("cp.async.bulk.wait_group.read 0;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -410,7 +410,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<0>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 0;"); + asm volatile("cp.async.bulk.wait_group.read 0;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -418,7 +418,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<0>() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<1>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 1;"); + asm volatile("cp.async.bulk.wait_group.read 1;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -426,7 +426,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<1>() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<2>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 2;"); + asm volatile("cp.async.bulk.wait_group.read 2;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -434,7 +434,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<2>() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<4>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 4;"); + asm volatile("cp.async.bulk.wait_group.read 4;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -443,7 +443,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<4>() { // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group __device__ __forceinline__ void cp_async_bulk_commit_group() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.commit_group;"); + asm volatile("cp.async.bulk.commit_group;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_commit_group is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -452,7 +452,7 @@ __device__ __forceinline__ void cp_async_bulk_commit_group() { // Proxy fence (bi-directional): __device__ __forceinline__ void fence_proxy_async() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("fence.proxy.async;"); + asm volatile("fence.proxy.async;" ::: "memory"); #else NVTE_DEVICE_ERROR("fence_proxy_async is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -460,7 +460,7 @@ __device__ __forceinline__ void fence_proxy_async() { __device__ __forceinline__ void fence_proxy_async_shared_cta() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("fence.proxy.async.shared::cta;"); + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); #else NVTE_DEVICE_ERROR("fence_proxy_async_shared_cta is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -752,7 +752,7 @@ __device__ __forceinline__ int32_t elect_one_sync(uint32_t mask = 0xFFFFFFFFu) { __device__ __forceinline__ void numbered_barrier_sync(uint32_t num_threads, uint32_t barrier_id = 1u) { - asm volatile("bar.sync %0, %1;\n" ::"r"(barrier_id), "r"(num_threads)); + asm volatile("bar.sync %0, %1;\n" ::"r"(barrier_id), "r"(num_threads) : "memory"); } __device__ __forceinline__ void fma_f32_f16(float &out, uint16_t const &a, uint16_t const &b, diff --git a/transformer_engine/pytorch/attention/dot_product_attention/backends.py b/transformer_engine/pytorch/attention/dot_product_attention/backends.py index f90fc46c032..bccb4f1e26f 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/backends.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/backends.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from importlib.metadata import version as get_pkg_version from importlib.metadata import PackageNotFoundError +import functools import inspect import os from typing import Any, Callable, Dict, List, Optional, Tuple, Union @@ -59,7 +60,11 @@ fa_prepare_fwd, ) from transformer_engine.pytorch.jit import no_torch_dynamo -from transformer_engine.pytorch.dynamo.custom_op import TensorOrQuantized +from transformer_engine.pytorch.dynamo.custom_op import ( + TensorOrQuantized, + register_custom_op_with_autograd, +) +from transformer_engine.pytorch.dynamo.tensor_spec import TensorSpec from transformer_engine.pytorch.attention.dot_product_attention.context_parallel import ( attn_forward_func_with_cp, ) @@ -218,6 +223,27 @@ except (ValueError, TypeError): fa_utils.fa3_supports_softcap = False + +def _fa4_with_none_window_sentinel(func: Callable) -> Callable: + """Translate TE's ``-1`` unbounded-window sentinel to FA4's ``None``. + + Preserve other negative offsets, which can represent empty windows. Adapt the FA4 entry + points because shared FA2/FA3 callers still use ``-1``. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + window = kwargs.get("window_size") + if window is not None: + kwargs["window_size"] = tuple(None if bound == -1 else bound for bound in window) + for bound_name in ("window_size_left", "window_size_right"): + if kwargs.get(bound_name) == -1: + kwargs[bound_name] = None + return func(*args, **kwargs) + + return wrapper + + # Try to import Flash Attention v4 try: fa_utils.fa4_version = PkgVersion(get_pkg_version("flash-attn-4")) @@ -259,10 +285,12 @@ else: # Unlike versions 2 and 3, FlashAttention 4 registers no custom ops: it builds # its kernels through the CUTLASS DSL as it runs. Keep it an eager island. - flash_attn_func_v4 = no_torch_dynamo()(_flash_attn_func_v4) - flash_attn_varlen_func_v4 = no_torch_dynamo()(_flash_attn_varlen_func_v4) - _flash_attn_fwd_v4 = no_torch_dynamo()(_flash_attn_fwd_v4) - _flash_attn_bwd_v4 = no_torch_dynamo()(_flash_attn_bwd_v4) + flash_attn_func_v4 = no_torch_dynamo()(_fa4_with_none_window_sentinel(_flash_attn_func_v4)) + flash_attn_varlen_func_v4 = no_torch_dynamo()( + _fa4_with_none_window_sentinel(_flash_attn_varlen_func_v4) + ) + _flash_attn_fwd_v4 = no_torch_dynamo()(_fa4_with_none_window_sentinel(_flash_attn_fwd_v4)) + _flash_attn_bwd_v4 = no_torch_dynamo()(_fa4_with_none_window_sentinel(_flash_attn_bwd_v4)) fa_utils.v4_validate_head_dims = _fa4_validate_head_dims fa_utils.set_flash_attention_4_params() @@ -1523,12 +1551,14 @@ def convert_to_torch_float8(tensor, dtype): class FusedAttnFwdArgs: """Single-argument bag for the forward path of :class:`FusedAttnFunc`.""" - # --- Differentiable tensors (also passed positionally to autograd) --- - q: TensorOrQuantized - k: TensorOrQuantized - v: TensorOrQuantized + # --- Differentiable tensors --- + q: Optional[TensorOrQuantized] + k: Optional[TensorOrQuantized] + v: Optional[TensorOrQuantized] attn_bias: Optional[torch.Tensor] softmax_offset: Optional[torch.Tensor] + packed_qkv: Optional[torch.Tensor] + packed_kv: Optional[torch.Tensor] # --- Non-differentiable tensors --- cu_seqlens_q: torch.Tensor @@ -1537,8 +1567,6 @@ class FusedAttnFwdArgs: cu_seqlens_kv_padded: Optional[torch.Tensor] page_table_k: Optional[torch.Tensor] page_table_v: Optional[torch.Tensor] - packed_qkv: Optional[torch.Tensor] - packed_kv: Optional[torch.Tensor] # --- Attention config --- is_training: bool @@ -1581,6 +1609,8 @@ class FusedAttnBwdArgs: q: Optional[torch.Tensor] = None k: Optional[torch.Tensor] = None v: Optional[torch.Tensor] = None + packed_qkv: Optional[torch.Tensor] = None + packed_kv: Optional[torch.Tensor] = None out: Optional[torch.Tensor] = None cu_seqlens_q: Optional[torch.Tensor] = None cu_seqlens_kv: Optional[torch.Tensor] = None @@ -1644,6 +1674,8 @@ def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: self.rng_state, self.aux_bias, self.aux_softmax_offset, + self.packed_qkv, + self.packed_kv, ) = restore_from_func_ctx( ctx ) # pylint: disable=unbalanced-tuple-unpacking @@ -1658,6 +1690,16 @@ def aux_ctx_tensors(self) -> List[torch.Tensor]: return aux +def _fused_attn_qkv(args): + if args.q is None and args.packed_qkv is not None: + dim = -2 if "h3d" in args.qkv_layout else -3 + return args.packed_qkv.unbind(dim) + if args.k is None and args.packed_kv is not None: + dim = -2 if "h2d" in args.qkv_layout else -3 + return args.q, *args.packed_kv.unbind(dim) + return args.q, args.k, args.v + + def _fused_attn_forward_impl( args: FusedAttnFwdArgs, ) -> Tuple[Any, Optional[torch.Tensor], Tuple[Any, ...], Dict[str, Any]]: @@ -1670,7 +1712,7 @@ def _fused_attn_forward_impl( input / the output is replaced by ``None`` and named in ``ctx_attrs["saved_from"]``, so backward takes it from there instead. """ - q, k, v = args.q, args.k, args.v + q, k, v = _fused_attn_qkv(args) is_training = args.is_training fp8 = args.fp8 fp8_meta = args.fp8_meta @@ -1841,17 +1883,17 @@ def _fused_attn_forward_impl( # save q, k, v, o tensors fp8_tensors = (None, None, None, None) - f16_tensors = (None, None, None, None) + f16_tensors = (None, None, None, None, None, None) if is_bwd_fp8: if (qkv_type == "current" and _dpa_fp8_cs_o_in_f16) or qkv_type == "mxfp8": fp8_tensors = (q_fp8, k_fp8, v_fp8, None) - f16_tensors = (None, None, None, out_f16) + f16_tensors = (None, None, None, out_f16, None, None) elif qkv_type == "delayed" or (qkv_type == "current" and not _dpa_fp8_cs_o_in_f16): fp8_tensors = (q_fp8, k_fp8, v_fp8, out_fp8) else: if is_input_fp8: q, k, v = combine_and_dequantize(qkv_layout, q_fp8, k_fp8, v_fp8) - f16_tensors = (q, k, v, out_f16) + f16_tensors = (q, k, v, out_f16, None, None) else: # q, k, v, out_: torch.Tensor; dtype = torch.float16 or torch.bfloat16 out_, aux_ctx_tensors, *max_logit = fused_attn_fwd( @@ -1892,7 +1934,14 @@ def _fused_attn_forward_impl( out_f16 = out_ out_ret = out_ fp8_tensors = (None, None, None, None) - f16_tensors = (q, k, v, out_f16) + f16_tensors = ( + args.q, + args.k, + args.v, + out_f16, + args.packed_qkv if args.q is None else None, + args.packed_kv if args.k is None else None, + ) nvtx_range_pop(f"{nvtx_label}") @@ -1919,6 +1968,8 @@ def _fused_attn_forward_impl( "k" if f16_tensors[1] is args.k else None, "v" if f16_tensors[2] is args.v else None, "out" if f16_tensors[3] is out_ret else None, + "packed_qkv" if f16_tensors[4] is not None else None, + "packed_kv" if f16_tensors[5] is not None else None, None, None, "attn_bias" if has_bias else None, @@ -2046,6 +2097,8 @@ def _fused_attn_setup_ctx( "q": fwd_args.q, "k": fwd_args.k, "v": fwd_args.v, + "packed_qkv": fwd_args.packed_qkv, + "packed_kv": fwd_args.packed_kv, "attn_bias": fwd_args.attn_bias, "softmax_offset": fwd_args.softmax_offset, "out": out, @@ -2059,6 +2112,8 @@ def _fused_attn_setup_ctx( k, v, out_f16, + packed_qkv, + packed_kv, softmax_stats, rng_state, aux_bias, @@ -2084,6 +2139,8 @@ def _fused_attn_setup_ctx( rng_state, aux_bias, aux_softmax_offset, + packed_qkv, + packed_kv, ) @@ -2111,7 +2168,8 @@ def _fused_attn_backward_impl( else: d_out_fp8 = args.dO_quantizer(d_out) q_fp8, k_fp8, v_fp8, out_fp8 = args.q_fp8, args.k_fp8, args.v_fp8, args.out_fp8 - q, k, v, out = args.q, args.k, args.v, args.out + q, k, v = _fused_attn_qkv(args) + out = args.out cu_seqlens_q, cu_seqlens_kv = args.cu_seqlens_q, args.cu_seqlens_kv cu_seqlens_q_padded, cu_seqlens_kv_padded = args.cu_seqlens_q_padded, args.cu_seqlens_kv_padded aux_ctx_tensors = args.aux_ctx_tensors() @@ -2369,6 +2427,176 @@ def backward(ctx, d_out, *_args): return (*_fused_attn_backward_impl(bwd_args), None) +def _fused_attn_q_shape(args) -> Tuple[int, ...]: + if args.q is not None: + return tuple(args.q.shape) + shape = list(args.packed_qkv.shape) + del shape[-2 if "h3d" in args.qkv_layout else -3] + return tuple(shape) + + +def _fused_attn_stats_shape( + args: FusedAttnFwdArgs, q_format: str, q_shape: Tuple[int, ...] +) -> Tuple[int, ...]: + """Shape of the softmax stats auxiliary tensor cuDNN returns.""" + if q_format == "thd": + num_heads = q_shape[1] + sm = get_device_compute_capability() + if sm >= (9, 0) and sm != (12, 0): + return (q_shape[0], num_heads, 1) + batch_size = args.cu_seqlens_q.shape[0] - 1 + return (batch_size, num_heads, args.max_seqlen_q, 1) + if q_format == "bshd": + batch_size, num_heads = q_shape[0], q_shape[2] + else: + batch_size, num_heads = q_shape[1], q_shape[2] + return (batch_size, num_heads, args.max_seqlen_q, 1) + + +def _fused_attn_forward_fake( + args: FusedAttnFwdArgs, +) -> Tuple[TensorSpec, Optional[TensorSpec], Tuple[Any, ...], Dict[str, Any]]: + """Data-free twin of :func:`_fused_attn_forward_impl` (non-FP8 only).""" + q = args.q if args.q is not None else args.packed_qkv + v = args.v + if v is None: + v = args.packed_qkv if args.packed_qkv is not None else args.packed_kv + q_shape = _fused_attn_q_shape(args) + _, o_format, _ = dpa_utils.get_qkv_format(args.qkv_layout) + out = TensorSpec(shape=(*q_shape[:-1], v.shape[-1]), dtype=q.dtype, device=q.device) + max_logit = None + if args.return_max_logit: + max_logit = TensorSpec(shape=(q_shape[-2],), dtype=q.dtype, device=q.device) + softmax_stats = TensorSpec( + shape=_fused_attn_stats_shape(args, o_format, q_shape), dtype=torch.float32, device=q.device + ) + rng_state = TensorSpec(shape=(2,), dtype=torch.int64, device=q.device) + has_bias = args.attn_bias_type not in ["no_bias", "alibi"] and args.attn_bias is not None + has_softmax_offset = args.softmax_type != "vanilla" and args.softmax_offset is not None + tensors_to_save = (*(None,) * 10, softmax_stats, rng_state, None, None) + saved_from = ( + *(None,) * 4, + "q", + "k", + "v", + "out", + "packed_qkv" if args.q is None and args.packed_qkv is not None else None, + "packed_kv" if args.k is None and args.packed_kv is not None else None, + None, + None, + "attn_bias" if has_bias else None, + "softmax_offset" if has_softmax_offset else None, + ) + ctx_attrs = { + "fp8": False, + "is_input_fp8": False, + "saved_from": saved_from, + "fused_attention_backend": args.fused_attention_backend, + "nominal_dtype": q.dtype, + "qkv_type": None, + "qkv_layout": args.qkv_layout, + "original_qkv_layout": args.qkv_layout, + "o_format": o_format, + "qkv_scale_inv_format": None, + "QKV_quantizer": None, + "O_quantizer": None, + "S_quantizer": None, + "dQKV_quantizer": None, + "dO_quantizer": None, + "dP_quantizer": None, + } + return out, max_logit, tensors_to_save, ctx_attrs + + +def _fused_attn_pack_grad(grad, dim, count): + """Recover the packed allocation produced by native attention backward.""" + shape, strides = list(grad.shape), list(grad.stride()) + packed_stride = shape[dim + 1] * strides[dim + 1] + shape.insert(dim + 1, count) + strides.insert(dim + 1, packed_stride) + return grad.as_strided(shape, strides) + + +def _fused_attn_backward_op_impl( + args: FusedAttnBwdArgs, +) -> Tuple[Optional[torch.Tensor], ...]: + dq, dk, dv, d_bias, d_softmax_offset = _fused_attn_backward_impl(args) + dqkv, dkv = None, None + if args.packed_qkv is not None: + dim = -2 if "h3d" in args.dqkv_layout else -3 + dqkv = _fused_attn_pack_grad(dq, dim, 3) + dq, dk, dv = None, None, None + elif args.packed_kv is not None: + dim = -2 if "h2d" in args.dqkv_layout else -3 + dkv = _fused_attn_pack_grad(dk, dim, 2) + dk, dv = None, None + return dq, dk, dv, d_bias, d_softmax_offset, dqkv, dkv + + +def _fused_attn_backward_fake( + args: FusedAttnBwdArgs, +) -> Tuple[Optional[TensorSpec], ...]: + """Data-free twin of :func:`_fused_attn_backward_op_impl`.""" + dq, dk, dv, dqkv, dkv = ( + TensorSpec(shape=tuple(t.shape), dtype=t.dtype, device=t.device) if t is not None else None + for t in (args.q, args.k, args.v, args.packed_qkv, args.packed_kv) + ) + q = args.q if args.q is not None else args.packed_qkv + d_bias = None + if args.aux_bias is not None: + d_bias = TensorSpec(shape=tuple(args.aux_bias.shape), dtype=q.dtype, device=q.device) + d_softmax_offset = None + if args.aux_softmax_offset is not None: + d_softmax_offset = TensorSpec( + shape=(1, _fused_attn_q_shape(args)[-2], 1, 1), dtype=torch.float32, device=q.device + ) + return dq, dk, dv, d_bias, d_softmax_offset, dqkv, dkv + + +# Custom op used under ``torch.compile``. +_fused_attn_op = register_custom_op_with_autograd( + op_name="fused_attn", + input_tensors_for_grad=[ + "q", + "k", + "v", + "attn_bias", + "softmax_offset", + "packed_qkv", + "packed_kv", + ], + fwd_arg_type=FusedAttnFwdArgs, + fwd_impl=_fused_attn_forward_impl, + fwd_fake_impl=_fused_attn_forward_fake, + setup_context=_fused_attn_setup_ctx, + bwd_arg_type=FusedAttnBwdArgs, + bwd_impl=_fused_attn_backward_op_impl, + bwd_fake_impl=_fused_attn_backward_fake, + fwd_tags=(torch.Tag.nondeterministic_seeded,), +) + + +def _needs_eager_fused_attention(call: Dict[str, Any]) -> Optional[str]: + """Why this FusedAttention call has to run outside the graph, or None. + + `call` maps `FusedAttention.forward`'s parameter names to the arguments + this call passed, including `self`. + """ + if _fused_attn_op is None: + return "fused attention without custom-op support" + if call.get("fp8", False): + return "FP8 attention" + if call.get("cp_group") is not None: + return "context parallelism" + if call.get("score_mod") is not None: + return "score_mod" + if call["self"].use_FAv2_bwd: + return "NVTE_FUSED_ATTN_USE_FAv2_BWD" + if is_cpu_offload_enabled(): + return "CPU activation offloading" + return None + + class FusedAttention(torch.nn.Module): """Dot product attention using `cuDNN attention `_: @@ -2441,7 +2669,7 @@ def remove_extra_states_check(self, incompatible_keys): # pylint: disable=unuse self.register_load_state_dict_post_hook(remove_extra_states_check) - @no_torch_dynamo() + @no_torch_dynamo(when=_needs_eager_fused_attention) def forward( self, query_layer: torch.Tensor, @@ -2697,21 +2925,30 @@ def forward( return_max_logit=self.return_max_logit, rng_gen=None, fp8=fp8, - fp8_meta=fp8_meta, - quantizers=quantizers, + fp8_meta=fp8_meta if fp8 else None, + quantizers=quantizers if fp8 else None, fp8_output=fp8_output, bf16_backward=bf16_backward, layer_number=self.layer_number, ) with self.attention_dropout_ctx(): - output = FusedAttnFunc.apply( - query_layer, - key_layer, - value_layer, - core_attention_bias, - softmax_offset, - fwd_args, - ) + if torch.compiler.is_compiling() and _fused_attn_op is not None: + if packed_qkv is not None: + fwd_args.q, fwd_args.k, fwd_args.v = None, None, None + elif packed_kv is not None: + fwd_args.k, fwd_args.v = None, None + output = _fused_attn_op(fwd_args) + if not self.return_max_logit: + output = output[0] + else: + output = FusedAttnFunc.apply( + query_layer, + key_layer, + value_layer, + core_attention_bias, + softmax_offset, + fwd_args, + ) if self.return_max_logit: # ...hd -> ...(hd) diff --git a/transformer_engine/pytorch/attention/dot_product_attention/utils.py b/transformer_engine/pytorch/attention/dot_product_attention/utils.py index 3e11039c1f4..e8cbd8a34cd 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/utils.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/utils.py @@ -148,11 +148,13 @@ class FlashAttentionUtils: v4_is_installed = False fa4_version = PkgVersion("0") + v4_0_0_beta31 = PkgVersion("4.0.0b31") + v4_0_0_beta33 = PkgVersion("4.0.0b33") use_v4 = False # Set by a signature probe in backends.py; fail-closed default. fa3_supports_softcap = False v4_installation_steps = """\ -pip install flash-attn-4==4.0.0b11 nvidia-cutlass-dsl[cu13]""" +pip install flash-attn-4==4.0.0b33 nvidia-cutlass-dsl[cu13]==4.8.0""" v4_warning_printed = False # Set by backends.py if FA4 is installed; calls flash_attn.cute.interface._validate_head_dims # which raises AssertionError for unsupported (head_dim, head_dim_v) combinations. @@ -1286,6 +1288,32 @@ def _is_fa3_supported(num_heads, num_gqa_groups, head_dim_qk, head_dim_v, qkv_dt cp_comm_type, ) use_flash_attention_4 = False + # SM100/SM110 D=256 gained seqused_q/k support in b31 for forward and b33 for backward. + # THD all-gather always passes this metadata; other THD paths pass it only + # when pad_between_seqs=True. + fa4_uses_seqused = qkv_format == "thd" and ( + pad_between_seqs or (context_parallel and cp_comm_type == "all_gather") + ) + if ( # pylint: disable=too-many-boolean-expressions + use_flash_attention_4 + and FlashAttentionUtils.v4_is_installed + and (10, 0) <= device_compute_capability < (12, 0) + and head_dim_qk == head_dim_v == 256 + and fa4_uses_seqused + and ( + FlashAttentionUtils.fa4_version < FlashAttentionUtils.v4_0_0_beta31 + or ( + attention_params.requires_backward + and FlashAttentionUtils.fa4_version < FlashAttentionUtils.v4_0_0_beta33 + ) + ) + ): + logger.debug( + "Disabling FlashAttention 4 for SM100/SM110 head_dim=256 with seqused_q/k: " + "version %s does not support the requested execution direction.", + FlashAttentionUtils.fa4_version, + ) + use_flash_attention_4 = False if context_parallel and ( use_flash_attention_2 or use_flash_attention_3 or use_flash_attention_4 ): @@ -1765,11 +1793,15 @@ def _is_fa3_supported(num_heads, num_gqa_groups, head_dim_qk, head_dim_v, qkv_dt ) use_flash_attention_3 = False if use_flash_attention_4 and deterministic and FlashAttentionUtils.v4_is_installed: - # eval() does not disable autograd; only forward-only calls can use this kernel. - if attention_params.requires_backward and device_compute_capability[0] == 12: + # eval() does not disable autograd. SM12x backward and SM100/SM110 D=256 backward + # reject deterministic execution, so only forward-only calls can use those kernels. + if attention_params.requires_backward and ( + device_compute_capability[0] == 12 + or ((10, 0) <= device_compute_capability < (12, 0) and head_dim_qk == head_dim_v == 256) + ): logger.debug( - "Disabling FlashAttention 4 for deterministic backward on SM12x: the CuTe" - " SM12x backward kernel does not support deterministic execution." + "Disabling FlashAttention 4 because the selected backward kernel does not " + "support deterministic execution." ) use_flash_attention_4 = False if use_fused_attention and deterministic: @@ -3275,16 +3307,8 @@ def mxfp8_quantize_only(tensor_quantizer_pairs, src_format): "bshd", "sbhd", ), f"mxfp8_quantize_only only supports bshd/sbhd, got {src_format!r}." - _s_dim = {"bshd": 1, "sbhd": 0} - _d_dim = {"bshd": 3, "sbhd": 3} - fp8_tensors = [] for tensor, quantizer in tensor_quantizer_pairs: - original_shape = tensor.shape - rs_shape = list(original_shape) - rs_shape[_d_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE - cs_shape = list(original_shape) - cs_shape[_s_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE if src_format == "bshd": t2d = tensor.view(*tensor.shape[:2], -1) else: @@ -3293,41 +3317,119 @@ def mxfp8_quantize_only(tensor_quantizer_pairs, src_format): quantizer.optimize_for_gemm = False fp8_2d = quantizer(t2d) quantizer.optimize_for_gemm = orig_optimize - # Re-wrap with the original 4D SBHD/BSHD shape so that shape[-1] equals the per-head - # dimension (matching Q's wrapper shape) and fused_attn_bwd produces 4D dkv that - # matches key/value's expected gradient shape in _KFQuantizeKVForAttn.backward. - fp8_t = MXFP8Tensor( - shape=original_shape, - dtype=tensor.dtype, - rowwise_data=( - fp8_2d._rowwise_data.view(original_shape) - if fp8_2d._rowwise_data is not None - else None - ), - rowwise_scale_inv=( - fp8_2d._rowwise_scale_inv.view(rs_shape) - if fp8_2d._rowwise_scale_inv is not None - else None - ), - columnwise_data=( - fp8_2d._columnwise_data.view(original_shape) - if fp8_2d._columnwise_data is not None - else None - ), - columnwise_scale_inv=( - fp8_2d._columnwise_scale_inv.view(cs_shape) - if fp8_2d._columnwise_scale_inv is not None - else None - ), - quantizer=quantizer, - requires_grad=False, - fp8_dtype=fp8_2d._fp8_dtype, - with_gemm_swizzled_scales=False, + fp8_tensors.append( + _mxfp8_tensor_in_src_format(fp8_2d, tensor.shape, tensor.dtype, quantizer, src_format) ) - fp8_tensors.append(fp8_t) return fp8_tensors +def _mxfp8_tensor_in_src_format(fp8_2d, shape, dtype, quantizer, src_format): + """Wrap the MXFP8 quantization of an attention input's 2D view with the input's shape. + + Re-wrap with the original 4D SBHD/BSHD shape so that shape[-1] equals the per-head + dimension (matching Q's wrapper shape) and fused_attn_bwd produces 4D dkv that + matches key/value's expected gradient shape in _KFQuantizeKVForAttn.backward. + """ + _s_dim = {"bshd": 1, "sbhd": 0} + _d_dim = {"bshd": 3, "sbhd": 3} + rs_shape = list(shape) + rs_shape[_d_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE + cs_shape = list(shape) + cs_shape[_s_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE + return MXFP8Tensor( + shape=shape, + dtype=dtype, + rowwise_data=( + fp8_2d._rowwise_data.view(shape) if fp8_2d._rowwise_data is not None else None + ), + rowwise_scale_inv=( + fp8_2d._rowwise_scale_inv.view(rs_shape) + if fp8_2d._rowwise_scale_inv is not None + else None + ), + columnwise_data=( + fp8_2d._columnwise_data.view(shape) if fp8_2d._columnwise_data is not None else None + ), + columnwise_scale_inv=( + fp8_2d._columnwise_scale_inv.view(cs_shape) + if fp8_2d._columnwise_scale_inv is not None + else None + ), + quantizer=quantizer, + requires_grad=False, + fp8_dtype=fp8_2d._fp8_dtype, + with_gemm_swizzled_scales=False, + ) + + +def fused_mla_kv_rope_mxfp8( + kv, k_rope, cos, sin, v_head_dim, key_quantizer=None, value_quantizer=None +): + """MLA key/value split, key RoPE and MXFP8 quantization in one kernel. + + Splits ``kv`` into the key dimensions without RoPE and the value, appends ``k_rope`` with + RoPE applied to the key, and quantizes key and value as + :func:`mxfp8_quantize_only` (``src_format="sbhd"``) does, with byte-identical results, + without writing key and value in BF16 first. The RoPE is that of Megatron-LM's MLA + (interleaved rotation pairs in, the two halves out), with its roundings; see + ``nvte_fused_mla_kv_rope_mxfp8``. Supports the head dimensions of DeepSeek-V3 (128 key + dimensions without RoPE, 64 RoPE dimensions, 128 value dimensions) on SM 10.0+. + + Parameters + ---------- + kv : torch.Tensor + [s, b, h, k_dim + v_head_dim] BF16: per head, the key dimensions without RoPE and then + the value. ``s`` must be a multiple of 128 and ``h`` even. + k_rope : torch.Tensor + [s, b, 1, emb_dim] or [s, b, emb_dim] BF16 key RoPE dimensions shared by the heads, + rotation pairs interleaved. May be a strided view, such as a slice of a projection + output. + cos, sin : torch.Tensor + [max_s, 1, 1, emb_dim] or [max_s, emb_dim], both BF16 or both FP32; row t is used for + token t. + v_head_dim : int + Value dimensions per head in ``kv``. + key_quantizer, value_quantizer : MXFP8Quantizer, optional + E4M3 quantizers with row-wise and column-wise usage; created if not given. + + Returns + ------- + key, value : MXFP8Tensor + [s, b, h, k_dim + emb_dim] and [s, b, h, v_head_dim], data and scale_invs in SBHD + layout; NOT yet BHSD-permuted or swizzled (see :func:`mxfp8_transpose_swizzle`). + """ + s, b, h, kv_dim = kv.shape + if s % (4 * MXFP8_BLOCK_SCALING_SIZE) != 0: + raise ValueError( + "fused_mla_kv_rope_mxfp8 requires a sequence length that is a multiple of " + f"{4 * MXFP8_BLOCK_SCALING_SIZE}, got {s}." + ) + quantizers = [ + ( + quantizer + if quantizer is not None + else MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True) + ) + for quantizer in (key_quantizer, value_quantizer) + ] + orig_optimize = [quantizer.optimize_for_gemm for quantizer in quantizers] + for quantizer in quantizers: + quantizer.optimize_for_gemm = False + try: + key_2d, value_2d = tex.fused_mla_kv_rope_mxfp8( + kv, k_rope, cos, sin, v_head_dim, quantizers[0], quantizers[1] + ) + finally: + for quantizer, optimize in zip(quantizers, orig_optimize): + quantizer.optimize_for_gemm = optimize + key_shape = (s, b, h, kv_dim - v_head_dim + k_rope.shape[-1]) + value_shape = (s, b, h, v_head_dim) + return ( + _mxfp8_tensor_in_src_format(key_2d, key_shape, kv.dtype, quantizers[0], "sbhd"), + _mxfp8_tensor_in_src_format(value_2d, value_shape, kv.dtype, quantizers[1], "sbhd"), + ) + + def mxfp8_transpose_swizzle(fp8_tensors, src_format): """Phase 2 of mxfp8_quantize_fast_path: batched BHSD-transpose + GEMM-swizzle. diff --git a/transformer_engine/pytorch/cross_entropy.py b/transformer_engine/pytorch/cross_entropy.py index ebbf30b965a..8ae3b853801 100644 --- a/transformer_engine/pytorch/cross_entropy.py +++ b/transformer_engine/pytorch/cross_entropy.py @@ -4,7 +4,7 @@ """Cross Entropy Loss API""" -from typing import Optional +from typing import Optional, Tuple, Union import warnings import torch @@ -32,6 +32,7 @@ def forward( dist_process_group=None, ignore_idx=-100, overwrite_input=False, + return_argmax=False, ): """Compute the loss and save the input and softmax statistics for backward.""" @@ -43,6 +44,7 @@ def forward( n_non_ignore, rank, world_size, + argmax, ) = triton_cross_entropy.cross_entropy_forward( inp, target, @@ -51,6 +53,7 @@ def forward( dist_process_group, ignore_idx, overwrite_input, + return_argmax, ) tensors_to_save = (saved_input.detach(), stats, target) if reduce_loss: @@ -63,10 +66,13 @@ def forward( ctx.ignore_idx = ignore_idx ctx.overwrite_input = overwrite_input ctx.did_backward = False + if return_argmax: + ctx.mark_non_differentiable(argmax) + return loss, argmax return loss @staticmethod - def backward(ctx, grad_output): + def backward(ctx, grad_output, *_): """Reconstruct the input gradient from the tensors saved during forward.""" if ctx.did_backward: @@ -91,7 +97,7 @@ def backward(ctx, grad_output): ) if ctx.overwrite_input: torch.autograd.graph.increment_version(saved_input) - return grad_input, None, None, None, None, None, None + return grad_input, None, None, None, None, None, None, None def _validate_inputs( @@ -133,7 +139,8 @@ def _parallel_cross_entropy_overwrite_input( reduce_loss: bool, dist_process_group: Optional[torch.distributed.ProcessGroup], ignore_idx: int, -) -> torch.Tensor: + return_argmax: bool, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Run destructive cross entropy outside Torch Dynamo's compiled graph.""" return CrossEntropyFunction.apply( @@ -144,6 +151,7 @@ def _parallel_cross_entropy_overwrite_input( dist_process_group, ignore_idx, True, + return_argmax, ) @@ -157,8 +165,9 @@ def parallel_cross_entropy( is_cg_capturable: bool = False, *, overwrite_input: bool = False, + return_argmax: bool = False, _input: Optional[torch.Tensor] = None, -) -> torch.Tensor: +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Cross entropy loss with optional distributed reduction. Loss and derivative calculations use FP32 arithmetic for BF16 and FP32 @@ -169,6 +178,11 @@ def parallel_cross_entropy( must not read or reuse it after starting backward. Only one backward pass is supported for each loss result. + With ``return_argmax=True``, the forward pass also finds the index of the + largest logit of each row while it scans the row for the softmax maximum, + so callers that need the predicted tokens (for example an accuracy metric) + do not have to read the logits again. + Parameters ---------- inp : torch.Tensor @@ -191,11 +205,18 @@ def parallel_cross_entropy( Allow ``inp`` to be overwritten during backward. The input must be contiguous and cannot be reused afterward. This mode is incompatible with ``torch.compile`` and will result in a graph break if used in that context. + return_argmax : bool, default = False + Also return the index of each row's largest logit in the full vocabulary, + as ``torch.argmax`` over the unsharded logits returns it: the first index + if several logits are equal. Rows with an ignored target get one too. + As with ``torch.argmax``, a NaN logit counts as the maximum: a row with + NaN logits gets the index of its first NaN (and a NaN loss). Returns ------- - torch.Tensor - The computed loss. + torch.Tensor or tuple of torch.Tensor + The computed loss. With ``return_argmax=True``, the tuple ``(loss, argmax)``, + where ``argmax`` is an int64 tensor with the shape ``inp.shape[:-1]``. """ if _input is not None: @@ -221,6 +242,7 @@ def parallel_cross_entropy( reduce_loss, dist_process_group, ignore_idx, + return_argmax, ) return CrossEntropyFunction.apply( inp, @@ -230,4 +252,5 @@ def parallel_cross_entropy( dist_process_group, ignore_idx, overwrite_input, + return_argmax, ) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 955763f43b1..5a4378ea328 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -570,6 +570,10 @@ at::Tensor fused_qkv_rope_backward(const at::Tensor &q_grad_out, const at::Tenso const NVTE_QKV_Format qkv_format, const bool interleaved, const int cp_size, const int cp_rank); +std::tuple fused_mla_kv_rope_mxfp8( + const at::Tensor &kv, const at::Tensor &k_rope, const at::Tensor &cos, const at::Tensor &sin, + const int64_t v_head_dim, py::handle key_quantizer, py::handle value_quantizer); + /*************************************************************************************************** * Miscellaneous **************************************************************************************************/ diff --git a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp index 4392fa4b433..aea334972c2 100644 --- a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp +++ b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp @@ -6,6 +6,7 @@ #include "../extensions.h" #include "common.h" +#include "pybind.h" namespace transformer_engine::pytorch { @@ -298,4 +299,68 @@ at::Tensor fused_qkv_rope_backward(const at::Tensor &q_grad_out, const at::Tenso return qkv_grad_input; } +std::tuple fused_mla_kv_rope_mxfp8( + const at::Tensor &kv, const at::Tensor &k_rope, const at::Tensor &cos, const at::Tensor &sin, + const int64_t v_head_dim, py::handle key_quantizer, py::handle value_quantizer) { + TORCH_CHECK(kv.is_cuda(), "expected kv on a CUDA device"); + const auto device = kv.device(); + TORCH_CHECK(k_rope.device() == device && cos.device() == device && sin.device() == device, + "expected kv, k_rope, cos and sin on the same device"); + at::cuda::CUDAGuard device_guard(device); + TORCH_CHECK(kv.dim() == 4, "expected kv of shape [s, b, h, k_dim + v_dim], got ", kv.sizes()); + const int64_t s = kv.size(0); + const int64_t b = kv.size(1); + const int64_t h = kv.size(2); + const int64_t emb_dim = k_rope.size(-1); + TORCH_CHECK(k_rope.dim() >= 3 && k_rope.size(0) == s && k_rope.size(1) == b && + k_rope.numel() == s * b * emb_dim, + "expected k_rope of shape [s, b, 1, emb_dim] or [s, b, emb_dim], got ", + k_rope.sizes()); + TORCH_CHECK(v_head_dim > 0 && v_head_dim < kv.size(3), "invalid v_head_dim ", v_head_dim, + " for kv of shape ", kv.sizes()); + const int64_t key_dim = kv.size(3) - v_head_dim + emb_dim; + + // k_rope is usually a slice of a wider projection output. Its rows are read in place + // when they are a fixed, 16-byte aligned stride apart. + at::Tensor k_rope_rows = k_rope; + int64_t k_rope_stride = b > 1 ? k_rope_rows.stride(1) : k_rope_rows.stride(0); + if (k_rope_rows.stride(-1) != 1 || + (b > 1 && k_rope_rows.stride(0) != b * k_rope_rows.stride(1)) || k_rope_stride < emb_dim || + k_rope_stride % 8 != 0 || reinterpret_cast(k_rope_rows.data_ptr()) % 16 != 0) { + k_rope_rows = k_rope.contiguous(); + k_rope_stride = emb_dim; + } + const auto kv_c = kv.contiguous(); + const auto cos_c = cos.contiguous(); + const auto sin_c = sin.contiguous(); + auto kv_cu = makeTransformerEngineTensor(kv_c); + auto k_rope_cu = makeTransformerEngineTensor( + k_rope_rows.data_ptr(), + std::vector{static_cast(s * b), static_cast(emb_dim)}, + GetTransformerEngineDType(k_rope_rows.scalar_type())); + auto cos_cu = makeTransformerEngineTensor(cos_c); + auto sin_cu = makeTransformerEngineTensor(sin_c); + + auto key_quantizer_cpp = convert_quantizer(key_quantizer); + auto value_quantizer_cpp = convert_quantizer(value_quantizer); + const auto *key_mxfp8 = dynamic_cast(key_quantizer_cpp.get()); + const auto *value_mxfp8 = dynamic_cast(value_quantizer_cpp.get()); + TORCH_CHECK(key_mxfp8 != nullptr && value_mxfp8 != nullptr, + "fused_mla_kv_rope_mxfp8 requires MXFP8 quantizers"); + TORCH_CHECK(!key_mxfp8->with_2d_quantization && !value_mxfp8->with_2d_quantization, + "fused_mla_kv_rope_mxfp8 does not support 2D MXFP8 quantization"); + const auto fake_dtype = GetTransformerEngineDType(kv.scalar_type()); + auto [key_cu, key_py] = key_quantizer_cpp->create_tensor( + {static_cast(s), static_cast(b * h * key_dim)}, fake_dtype, device); + auto [value_cu, value_py] = value_quantizer_cpp->create_tensor( + {static_cast(s), static_cast(b * h * v_head_dim)}, fake_dtype, device); + + NVTE_SCOPED_GIL_RELEASE({ + nvte_fused_mla_kv_rope_mxfp8(kv_cu.data(), k_rope_cu.data(), static_cast(k_rope_stride), + cos_cu.data(), sin_cu.data(), key_cu.data(), value_cu.data(), + at::cuda::getCurrentCUDAStream()); + }); + return {key_py, value_py}; +} + } // namespace transformer_engine::pytorch diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index 43f1d32b8a3..e93c01026a9 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -337,6 +337,8 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w UNFUSED, // Compute norm directly FULLY_FUSED, + // Compute norm directly, with quantization scales in compact format + FUSED_NORM_QUANT_UNSWIZZLED, // Compute norm and amax in high precision, then quantize to FP8 FUSED_NORM_AMAX_FP8, // Compute norm and amax in high precision, then quantize to NVFP4 @@ -346,10 +348,17 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && + auto mxfp8_quantizer_cpp = dynamic_cast(quantizer_cpp.get()); + NVTE_CHECK(mxfp8_quantizer_cpp != nullptr, "Could not cast to MXFP8 quantizer"); + // The fused MXFP8 kernels quantize 1D blocks and require full 128x128 tiles + if (!mxfp8_quantizer_cpp->with_2d_quantization && outer_size % 128 == 0 && inner_size % 128 == 0) { - // cuDNN MXFP8 kernel requires full 128x128 tiles - impl = Impl::FULLY_FUSED; + if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { + impl = Impl::FUSED_NORM_QUANT_UNSWIZZLED; + } else { + // Transformer Engine's fused RMSNorm + MXFP8 kernel + impl = Impl::FULLY_FUSED; + } } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && !transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { @@ -375,9 +384,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w // Output tensor TensorWrapper out_nvte; if (out.is_none()) { - if (impl == Impl::FULLY_FUSED) { - // FP8 has no special logic to optimize for GEMM, MXFP8 cuDNN - // kernel does not support GEMM swizzled scales + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { quantizer_cpp->optimize_for_gemm = false; } std::tie(out_nvte, out) = quantizer_cpp->create_tensor(shape, out_dtype); diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 2046408eba8..c7f70182f36 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -589,6 +589,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Fused Apply QKV RoPE FWD", py::call_guard()); m.def("fused_qkv_rope_backward", &transformer_engine::pytorch::fused_qkv_rope_backward, "Fused Apply QKV RoPE BWD", py::call_guard()); + m.def("fused_mla_kv_rope_mxfp8", &transformer_engine::pytorch::fused_mla_kv_rope_mxfp8, + "Split MLA key/value, apply RoPE to the key and quantize both to MXFP8", py::arg("kv"), + py::arg("k_rope"), py::arg("cos"), py::arg("sin"), py::arg("v_head_dim"), + py::arg("key_quantizer"), py::arg("value_quantizer")); // fused router transformer_engine::pytorch::init_router_bindings(m); diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index a87374095a7..5392316037f 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -17,8 +17,9 @@ The autograd-free API preserves nested tensor results; the autograd-wired API keeps its saved-tensor and context-metadata contract. -A ``torch.library`` custom op is narrower: it only accepts flat schema slots -(tensors plus opaque objects) and returns a flat ``Tensor[]``. +A ``torch.library`` custom op accepts schema slots (tensors plus opaque objects). +Forward returns a flat ``Tensor[]``; backward preserves each gradient field as +a ``Tensor`` or ``Tensor[]`` return. Bridging the two takes three parts (below): a parsed per-op *arg plan* maps the args dataclass onto the op's input slots; a per-trace *output plan*, parsed from @@ -37,6 +38,8 @@ * ``TENSOR_OR_QUANTIZED`` -- a field that may be a plain tensor, a bare quantized storage, or ``None``: three slots (the tensor, its flat inner buffers, and a ``__kind__`` tag) so a quantized tensor crosses as its buffers. + * Tensor kinds also support lists, with per-element gradients and metadata. + * ``SYMINT_LIST`` -- ``List[int]`` in a ``SymInt[]`` slot for dynamic splits. * ``SIMPLE`` -- every remaining simple value (scalars, enums, sizes, quantizers -- value-opaque constants baked into the graph -- and nested collections of them), gathered into one shared ``OpaqueValueBundle`` slot. @@ -70,7 +73,7 @@ whole tuple; otherwise ``grad_output`` receives the first output's grad), the container's optional ``setup_saved_tensors`` hook restores the saved tensors, then the *backward op* runs the real ``bwd_impl`` and returns the - flat grads (``bwd_fake_impl`` is its data-free fake). + structured grads (``bwd_fake_impl`` is its data-free fake). Two-tier op (``base`` + ``wrapper``), so a ``QuantizedTensor`` subclass can be an op *input*. The ``_base`` op carries the real schema + autograd; a custom op @@ -360,7 +363,7 @@ def _storage_unflatten(meta: "OpaqueValueBundle", tensors: List[torch.Tensor]) - inner = dict(zip(inner_names, tensors)) outer_shape = meta_dict.get("_outer_shape") stride = make_contiguous_strides_for(tuple(outer_shape)) if outer_shape is not None else None - return QuantizedTensorStorage.__tensor_unflatten__(inner, meta_dict, outer_shape, stride) + return meta_dict["cls"].__tensor_unflatten__(inner, meta_dict, outer_shape, stride) # --------------------------------------------------------------------------- # @@ -397,6 +400,7 @@ class _FieldKind(Enum): TENSOR = "tensor" # one ``Tensor`` / ``Tensor?`` slot TENSOR_OR_QUANTIZED = "tensor_or_quantized" # 3 slots: tensor / inner / meta + SYMINT_LIST = "symint_list" PROCESS_GROUP = "process_group" # c10d group name inside the shared bundle SIMPLE = "simple" # value carried verbatim inside the shared bundle UNSUPPORTED = "unsupported" # no slots; only a trivial value may cross @@ -437,6 +441,7 @@ class _FieldPlan: name: str kind: _FieldKind slots: Tuple[_SlotSpec, ...] + is_list: bool = False def _is_tensor_storage_union(annot: Any) -> bool: @@ -480,6 +485,22 @@ def _is_simple_annot(annot: Any) -> bool: def _parse_field(name: str, annot: Any) -> _FieldPlan: """Parse one field's annotation into its :class:`_FieldPlan`.""" + stripped, _ = _strip_optional(annot) + if get_origin(stripped) is list and len(get_args(stripped)) == 1: + element_annot = get_args(stripped)[0] + if element_annot is int: + return _FieldPlan(name, _FieldKind.SYMINT_LIST, (_SlotSpec(name, "SymInt[]"),)) + element = _parse_field(name, element_annot) + if element.is_list: + raise TypeError(f"field {name!r}: nested tensor lists are not supported") + if element.kind in (_FieldKind.TENSOR, _FieldKind.TENSOR_OR_QUANTIZED): + slots = tuple( + _SlotSpec( + slot.name, "Tensor[]" if slot.type_str.startswith("Tensor") else slot.type_str + ) + for slot in element.slots + ) + return _FieldPlan(name, element.kind, slots, is_list=True) if _is_tensor_storage_union(annot): slots = ( _SlotSpec(name, "Tensor?"), @@ -522,6 +543,10 @@ def _is_trivial(value: Any) -> bool: def _pack_tensor_or_quantized(field: _FieldPlan, value: Any, slots: Dict[str, Any]) -> None: """Fill a tensor-or-quantized field's three slots from its runtime value.""" + from ..tensor.storage.grouped_tensor_storage import ( + GroupedTensorStorage, + ) # pylint: disable=import-outside-toplevel + tensor_slot, inner_slot, meta_slot = (s.name for s in field.slots) if value is None: slots[tensor_slot] = None @@ -534,7 +559,7 @@ def _pack_tensor_or_quantized(field: _FieldPlan, value: Any, slots: Dict[str, An slots[tensor_slot] = value slots[inner_slot] = [] slots[meta_slot] = OpaqueValueBundle({_TQ_KIND_KEY: _TensorOrQuantizedKind.TENSOR}) - elif isinstance(value, QuantizedTensorStorage): + elif isinstance(value, (QuantizedTensorStorage, GroupedTensorStorage)): meta, tensors = _storage_flatten(value, {_TQ_KIND_KEY: _TensorOrQuantizedKind.STORAGE}) slots[tensor_slot] = None slots[inner_slot] = tensors @@ -542,7 +567,7 @@ def _pack_tensor_or_quantized(field: _FieldPlan, value: Any, slots: Dict[str, An else: raise TypeError( f"field {field.name!r} expected None, torch.Tensor, or " - f"QuantizedTensorStorage, got {type(value).__name__}" + f"QuantizedTensorStorage / GroupedTensorStorage, got {type(value).__name__}" ) @@ -558,6 +583,49 @@ def _unpack_tensor_or_quantized(field: _FieldPlan, slots: Dict[str, Any]) -> Any return _storage_unflatten(meta, slots[inner_slot]) +def _pack_tensor_list(field: _FieldPlan, values: Any, slots: Dict[str, Any]) -> None: + """Pack a tensor field's list using the scalar representation per element.""" + if field.kind is _FieldKind.TENSOR: + slots[field.slots[0].name] = [_encode_none(value) for value in values] + return + outer, inner, items, counts = [], [], [], [] + tensor_slot, inner_slot, meta_slot = (s.name for s in field.slots) + for value in values: + element_slots = {} + _pack_tensor_or_quantized(field, value, element_slots) + outer.append(_encode_none(element_slots[tensor_slot])) + tensors = element_slots[inner_slot] + inner.extend(tensors) + counts.append(len(tensors)) + items.append(element_slots[meta_slot]) + slots[tensor_slot] = outer + slots[inner_slot] = inner + slots[meta_slot] = OpaqueValueBundle({"items": items, "counts": counts}) + + +def _unpack_tensor_list(field: _FieldPlan, slots: Dict[str, Any]) -> List[Any]: + """Inverse of :func:`_pack_tensor_list`.""" + if field.kind is _FieldKind.TENSOR: + return [_decode_none(value) for value in slots[field.slots[0].name]] + tensor_slot, inner_slot, meta_slot = (s.name for s in field.slots) + bundle = slots[meta_slot] + values = [] + cursor = 0 + for i, (meta, count) in enumerate(zip(bundle["items"], bundle["counts"])): + values.append( + _unpack_tensor_or_quantized( + field, + { + tensor_slot: _decode_none(slots[tensor_slot][i]), + inner_slot: slots[inner_slot][cursor : cursor + count], + meta_slot: meta, + }, + ) + ) + cursor += count + return values + + @dataclasses.dataclass(frozen=True) class _ArgPlan: """The parsed plan for one args dataclass: the single source of truth for @@ -585,17 +653,17 @@ def tensor_field_names(self) -> Tuple[str, ...]: if f.kind in (_FieldKind.TENSOR, _FieldKind.TENSOR_OR_QUANTIZED) ) - def tensor_or_quantized_offsets(self) -> List[int]: + def tensor_or_quantized_offsets(self) -> List[Tuple[int, bool]]: """Start offset of each tensor-or-quantized slot group. Derived from ``fields`` on demand -- used once per registration, for the subclass-flattening dispatch wiring. """ - offsets: List[int] = [] + offsets: List[Tuple[int, bool]] = [] offset = 0 for field in self.fields: if field.kind is _FieldKind.TENSOR_OR_QUANTIZED: - offsets.append(offset) + offsets.append((offset, field.is_list)) offset += len(field.slots) return offsets @@ -630,7 +698,12 @@ def pack(self, obj: Any) -> Dict[str, Any]: simple: Dict[str, Any] = {} for field in self.fields: value = getattr(obj, field.name, None) + if field.is_list: + _pack_tensor_list(field, value, slots) + continue match field.kind: + case _FieldKind.SYMINT_LIST: + slots[field.slots[0].name] = list(value) case _FieldKind.TENSOR: slots[field.slots[0].name] = value case _FieldKind.TENSOR_OR_QUANTIZED: @@ -664,7 +737,12 @@ def unpack(self, slots: Dict[str, Any]) -> Any: kwargs: Dict[str, Any] = {} bundle = slots.get(_SIMPLE_META_SLOT) for field in self.fields: + if field.is_list: + kwargs[field.name] = _unpack_tensor_list(field, slots) + continue match field.kind: + case _FieldKind.SYMINT_LIST: + kwargs[field.name] = list(slots[field.slots[0].name]) case _FieldKind.TENSOR: kwargs[field.name] = slots[field.slots[0].name] case _FieldKind.TENSOR_OR_QUANTIZED: @@ -741,7 +819,12 @@ def _spec_view(obj: Any, tensor_field_names: Sequence[str]) -> Any: overrides: Dict[str, Any] = {} for name in tensor_field_names: value = getattr(obj, name, None) - if value is not None and not isinstance(value, TensorSpec): + if isinstance(value, (list, tuple)): + overrides[name] = [ + to_tensor_spec(v) if v is not None and not isinstance(v, TensorSpec) else v + for v in value + ] + elif value is not None and not isinstance(value, TensorSpec): overrides[name] = to_tensor_spec(value) if not overrides: return obj @@ -756,22 +839,35 @@ def _spec_view(obj: Any, tensor_field_names: Sequence[str]) -> Any: # --------------------------------------------------------------------------- # -def _spec_slot_count(spec: Optional[TensorSpec]) -> int: +@dataclasses.dataclass(frozen=True) +class SavedTensorRef: + """Saved input field, or user output when ``source`` is None.""" + + source: Optional[str] + index: Optional[int] = None + + def resolve(self, args, outputs): + """Resolve the reference before saving through autograd.""" + value = outputs if self.source is None else getattr(args, self.source) + return value if self.index is None else value[self.index] + + +def _spec_slot_count(spec: Optional[Union[TensorSpec, SavedTensorRef]]) -> int: """Flat ``Tensor[]`` slots the value for ``spec`` occupies.""" - if spec is None: + if spec is None or isinstance(spec, SavedTensorRef): return 1 return len(spec.inner_names()) def _flatten_value( - value: Optional[Union[torch.Tensor, QuantizedTensorStorage, TensorSpec]], + value: Optional[Union[torch.Tensor, QuantizedTensorStorage, TensorSpec, SavedTensorRef]], ) -> List[torch.Tensor]: """Return the flat ``Tensor[]`` slots that represent one op output ``value``. Pack-side inverse of :meth:`_OutputPlan.user_outputs`; the slot count matches :func:`_spec_slot_count`. """ - if value is None: + if value is None or isinstance(value, SavedTensorRef): return [_encode_none(None)] if isinstance(value, TensorSpec): return [_encode_none(t) for t in value.create_inner_tensors()] @@ -831,27 +927,30 @@ def _pack_fwd_result(result: Any) -> List[torch.Tensor]: return flat -def _pack_bwd_result( - grads: Any, num_grad_inputs: Optional[int], op_qualname: str -) -> List[torch.Tensor]: - """Pack a backward-impl return tuple into the op's ``Tensor[]`` payload. - - Each grad occupies exactly one slot (validated against ``num_grad_inputs`` - when given); a :class:`TensorSpec` grad is materialized into a single tensor. - """ - grads = list(grads) +def _pack_bwd_result(grads: Any, num_grad_inputs: Optional[int], op_qualname: str) -> Any: + """Materialize gradients while preserving one return per input field.""" if num_grad_inputs is not None and len(grads) != num_grad_inputs: raise RuntimeError( f"{op_qualname} expected bwd_impl to return {num_grad_inputs} grads " f"(one per input_tensors_for_grad entry), got {len(grads)}" ) - out: List[torch.Tensor] = [] - for g in grads: - if isinstance(g, TensorSpec): - out.append(_encode_none(g.create_tensor())) - else: - out.append(_encode_none(g)) - return out + + def materialize(grad): + return _encode_none(grad.create_tensor() if isinstance(grad, TensorSpec) else grad) + + out = tuple( + [materialize(g) for g in grad] if isinstance(grad, (list, tuple)) else materialize(grad) + for grad in grads + ) + return out if out else None + + +def _restore_grad_shape(grad, shape): + """Decode a gradient and restore its original input shape.""" + grad = _decode_none(grad) + if grad is not None and shape is not None and grad.shape != shape: + grad = grad.view(shape) + return grad @dataclasses.dataclass(frozen=True) @@ -867,7 +966,7 @@ class _OutputPlan: """ user_specs: Tuple[Optional[TensorSpec], ...] - saved_specs: Tuple[Optional[TensorSpec], ...] + saved_specs: Tuple[Optional[Union[TensorSpec, SavedTensorRef]], ...] ctx_attrs: Dict[str, Any] user_ranges: Tuple[Tuple[int, int], ...] saved_start: int @@ -909,13 +1008,17 @@ def user_outputs(self, flat: Sequence[Optional[torch.Tensor]]) -> List[Any]: for spec, (start, stop) in zip(self.user_specs, self.user_ranges) ] - def saved_tensors(self, flat: Sequence[Optional[torch.Tensor]]) -> List[Any]: + def saved_tensors(self, flat: Sequence[Optional[torch.Tensor]], args, outputs) -> List[Any]: """Rebuild the saved-for-backward tensors from the op's flat return.""" values: List[Any] = [] cursor = self.saved_start for spec in self.saved_specs: n = _spec_slot_count(spec) - values.append(self._assemble(spec, flat, cursor, cursor + n)) + values.append( + spec.resolve(args, outputs) + if isinstance(spec, SavedTensorRef) + else self._assemble(spec, flat, cursor, cursor + n) + ) cursor += n return values @@ -974,7 +1077,8 @@ def _register_base_op( plan: _ArgPlan, impl: Callable[[Any], Any], fake_impl: Callable[[Any], Any], - pack_result: Callable[[Any], List[torch.Tensor]], + pack_result: Callable[[Any], Any], + tags: Sequence[torch.Tag] = (), ) -> Any: """Define the op via ``torch.library.custom_op`` with the real ``impl`` + the ``fake_impl`` (spec), returning the ``CustomOpDef``. @@ -984,17 +1088,17 @@ def _register_base_op( ``pack_result``. """ - def _impl(*flat: Any) -> List[torch.Tensor]: + def _impl(*flat: Any) -> Any: obj = plan.unpack(dict(zip(plan.slot_names, flat))) return pack_result(impl(obj)) - def _fake(*flat: Any) -> List[torch.Tensor]: + def _fake(*flat: Any) -> Any: obj = plan.unpack(dict(zip(plan.slot_names, flat))) spec_obj = _spec_view(obj, plan.tensor_field_names()) return pack_result(fake_impl(spec_obj)) op = torch.library.custom_op( - f"{_TE_OP_NAMESPACE}::{op_name}", _impl, mutates_args=(), schema=schema_str + f"{_TE_OP_NAMESPACE}::{op_name}", _impl, mutates_args=(), schema=schema_str, tags=tags ) op.register_fake(_fake) _mark_effectful(op) @@ -1018,18 +1122,26 @@ def _register_autograd_for_op( the saved tuple + ``ctx_attrs`` to the module's ``setup_context`` and stashes the plan on ``ctx`` so backward can slice its grads per user output. """ + tensor_list_slots = tuple( + i + for i, slot in enumerate(s for f in fwd_plan.fields for s in f.slots) + if slot.type_str == "Tensor[]" + ) bwd_takes_grad_tuple = any(f.name == "grad_outputs" for f in bwd_plan.fields) def _setup_context(ctx, inputs, output): + # Autograd treats empty SymInt[] inputs as tensor lists too. ctx.fwd_tensor_list_lengths = { - i: len(value) for i, value in enumerate(inputs) if isinstance(value, list) + i: len(value) + for i, value in enumerate(inputs) + if i in tensor_list_slots or (isinstance(value, list) and not value) } fwd_obj = fwd_plan.unpack(dict(zip(fwd_plan.slot_names, inputs))) spec_obj = _spec_view(fwd_obj, fwd_plan.tensor_field_names()) out_plan = _OutputPlan.parse(fwd_fake_impl(spec_obj)) user_outputs = out_plan.user_outputs(output) - saved_list = out_plan.saved_tensors(output) + saved_list = out_plan.saved_tensors(output, fwd_obj, user_outputs) bwd_obj = bwd_plan.arg_type() tensors_to_save_from_setup = setup_context_user( @@ -1048,7 +1160,13 @@ def _setup_context(ctx, inputs, output): # rederive shapes lossily (e.g. rank-1 inputs come back rank-2), so the # returned grads are viewed back to the true input shapes below. ctx.grad_input_shapes = { - pos: inputs[pos].shape for pos in grad_targets if isinstance(inputs[pos], torch.Tensor) + pos: ( + [value.shape for value in inputs[pos]] + if pos in tensor_list_slots + else inputs[pos].shape + ) + for pos in grad_targets + if pos in tensor_list_slots or isinstance(inputs[pos], torch.Tensor) } def _autograd_backward(ctx, *grad_outputs): @@ -1064,7 +1182,7 @@ def _autograd_backward(ctx, *grad_outputs): bwd_obj.grad_output = user_grads[0] kwargs = bwd_plan.pack(bwd_obj) bwd_args_flat = [kwargs[name] for name in bwd_plan.slot_names] - grads = [_decode_none(g) for g in bwd_op(*bwd_args_flat)] + grads = bwd_op(*bwd_args_flat) ctx.backward_objects = None # One grad per input schema slot: default None, but a ``Tensor[]`` slot # (always recorded in ``fwd_tensor_list_lengths``) needs a @@ -1072,25 +1190,54 @@ def _autograd_backward(ctx, *grad_outputs): out: List[Any] = [None] * len(fwd_plan.slot_names) for pos, length in ctx.fwd_tensor_list_lengths.items(): out[pos] = [None] * length - for pos, g in zip(grad_targets, grads): - if g is not None: - shape = ctx.grad_input_shapes.get(pos) - if shape is not None and g.shape != shape: - g = g.view(shape) - out[pos] = g + for pos, grad in zip(grad_targets, grads or ()): + shapes = ctx.grad_input_shapes.get(pos) + out[pos] = ( + [_restore_grad_shape(g, s) for g, s in zip(grad, shapes, strict=True)] + if pos in tensor_list_slots + else _restore_grad_shape(grad, shapes) + ) ctx.grad_input_shapes = None return tuple(out) fwd_op.register_autograd(_autograd_backward, setup_context=_setup_context) +def _flatten_list_subclasses(new_args: List[Any], offset: int, subclass: type) -> None: + """Flatten subclass entries inside a tensor-or-quantized list slot.""" + outer = list(new_args[offset]) + if not any(isinstance(value, subclass) for value in outer): + return + bundle = new_args[offset + 2] + items = list(bundle["items"]) + counts = list(bundle["counts"]) + inner = [] + cursor = 0 + for i, value in enumerate(outer): + tensors = new_args[offset + 1][cursor : cursor + counts[i]] + cursor += counts[i] + if isinstance(value, subclass): + items[i], tensors = _storage_flatten( + value, {_TQ_KIND_KEY: _TensorOrQuantizedKind.STORAGE} + ) + outer[i] = _encode_none(None) + counts[i] = len(tensors) + inner.extend(tensors) + new_args[offset] = outer + new_args[offset + 1] = inner + new_args[offset + 2] = OpaqueValueBundle({"items": items, "counts": counts}) + + def _flatten_subclass_into_slots( - new_args: List[Any], slot_offsets: Sequence[int], subclass: type + new_args: List[Any], slot_offsets: Sequence[Tuple[int, bool]], subclass: type ) -> None: """Rewrite each tensor-or-quantized slot group whose ``Tensor?`` slot holds an instance of ``subclass`` into the storage layout (3 slots: name / tensors / meta). """ - for offset in slot_offsets: + for offset, is_list in slot_offsets: + if is_list: + _flatten_list_subclasses(new_args, offset, subclass) + continue val = new_args[offset] if not isinstance(val, subclass): continue @@ -1101,8 +1248,8 @@ def _flatten_subclass_into_slots( def _make_slot_forwarder( - base_op: Any, slot_offsets: Sequence[int], subclasses: Sequence[type] -) -> Callable[[Sequence[Any]], List[torch.Tensor]]: + base_op: Any, slot_offsets: Sequence[Tuple[int, bool]], subclasses: Sequence[type] +) -> Callable[[Sequence[Any]], Any]: """Return ``call(args)`` forwarding to ``base_op``, first flattening any ``subclasses`` instance sitting in the tensor-or-quantized slot groups at ``slot_offsets``. @@ -1114,7 +1261,7 @@ def _make_slot_forwarder( """ enabled = bool(slot_offsets) and bool(subclasses) - def call(args: Sequence[Any]) -> List[torch.Tensor]: + def call(args: Sequence[Any]) -> Any: if not enabled: return base_op(*args) new_args = list(args) @@ -1126,7 +1273,7 @@ def call(args: Sequence[Any]) -> List[torch.Tensor]: def _make_dispatch_rule( - forward: Callable[[Sequence[Any]], List[torch.Tensor]], + forward: Callable[[Sequence[Any]], Any], ) -> Callable[..., Any]: """Adapt a slot forwarder to the ``register_torch_dispatch`` signature.""" @@ -1142,31 +1289,37 @@ def _register_wrapper_op( wrapper_op_name: str, schema_str: str, base_op: Any, - slot_offsets: Sequence[int] = (), + slot_offsets: Sequence[Tuple[int, bool]] = (), subclasses: Sequence[type] = (), + tags: Sequence[torch.Tag] = (), ) -> Any: """Define the wrapper op via ``torch.library.custom_op``: forward to the base op through :func:`_make_slot_forwarder`. Returns the ``CustomOpDef``. """ forward = _make_slot_forwarder(base_op, slot_offsets, subclasses) - def _forward(*flat: Any) -> List[torch.Tensor]: + def _forward(*flat: Any) -> Any: return forward(flat) op_def = torch.library.custom_op( - f"{_TE_OP_NAMESPACE}::{wrapper_op_name}", _forward, mutates_args=(), schema=schema_str + f"{_TE_OP_NAMESPACE}::{wrapper_op_name}", + _forward, + mutates_args=(), + schema=schema_str, + tags=tags, ) op_def.register_fake(_forward) _mark_effectful(op_def) return op_def -def _all_quantized_tensor_subclasses() -> List[type]: - """Return every imported ``QuantizedTensor`` wrapper subclass.""" +def _all_tensor_subclasses() -> List[type]: + """Return grouped and imported quantized wrapper subclasses.""" import transformer_engine.pytorch.tensor # noqa: F401 pylint: disable=import-outside-toplevel,unused-import + from ..tensor.grouped_tensor import GroupedTensor # pylint: disable=import-outside-toplevel found: List[type] = [] - stack = list(QuantizedTensor.__subclasses__()) + stack = [GroupedTensor, *QuantizedTensor.__subclasses__()] while stack: cls = stack.pop() if cls not in found: @@ -1185,7 +1338,7 @@ class _RegisteredOp: wrapper_def: Any wrapper_op: Any - def __call__(self, args: Any) -> List[torch.Tensor]: + def __call__(self, args: Any) -> Any: """Pack the args dataclass into slots and call the wrapper op.""" kwargs = self.plan.pack(args) return self.wrapper_op(*[kwargs[name] for name in self.plan.slot_names]) @@ -1197,14 +1350,16 @@ def _register_op( arg_type: type, impl: Callable[[Any], Any], fake_impl: Callable[[Any], Any], - pack_result: Callable[[Any], List[torch.Tensor]], + pack_result: Callable[[Any], Any], + return_schema: str = "Tensor[]", + tags: Sequence[torch.Tag] = (), ) -> _RegisteredOp: """Define one two-tier custom op: the base kernel, the wrapper op that lets ``QuantizedTensor`` subclasses be inputs, and the passthrough registrations. """ plan = _parse_arg_type(arg_type) - schema = f"{plan.schema_str} -> Tensor[]" - subclasses = _all_quantized_tensor_subclasses() + schema = f"{plan.schema_str} -> {return_schema}" + subclasses = _all_tensor_subclasses() slot_offsets = plan.tensor_or_quantized_offsets() namespace = getattr(torch.ops, _TE_OP_NAMESPACE) @@ -1219,6 +1374,7 @@ def _register_op( impl=impl, fake_impl=fake_impl, pack_result=pack_result, + tags=tags, ) base_op = getattr(namespace, f"{name}_base") wrapper_def = _register_wrapper_op( @@ -1227,6 +1383,7 @@ def _register_op( base_op=base_op, slot_offsets=slot_offsets, subclasses=subclasses, + tags=tags, ) wrapper_op = getattr(namespace, name) @@ -1303,6 +1460,7 @@ def register_custom_op_with_autograd( bwd_impl: Callable[[Any], Any], fwd_fake_impl: Callable[[Any], Tuple[Any, ...]], bwd_fake_impl: Callable[[Any], Tuple[Any, ...]], + fwd_tags: Sequence[torch.Tag] = (), ) -> Optional[Callable[..., Any]]: """Register a TE module's forward + backward as torch custom ops. @@ -1310,6 +1468,8 @@ def register_custom_op_with_autograd( ``Function.apply`` under ``torch.compiler.is_compiling()`` that dispatches through the op and returns the user-facing outputs. + ``fwd_tags`` apply to both the base and wrapper forward operators. + ``fwd_arg_type`` and ``bwd_arg_type`` are ``@dataclass``es whose *field annotations* define the op schema (see the module docstring for the field <-> slot mapping). The caller builds a ``fwd_arg_type`` instance and @@ -1322,7 +1482,8 @@ def register_custom_op_with_autograd( * ``fwd_impl(fwd_args) -> (*user_outputs, tensors_to_save, ctx_attrs)`` -- the real forward. ``user_outputs``: op outputs (tensor / quantized / ``None``); - ``tensors_to_save``: list/tuple (or ``None``) of tensors for backward; + ``tensors_to_save``: list/tuple (or ``None``) of tensors or + :class:`SavedTensorRef` entries, resolved before ``setup_context``; ``ctx_attrs``: dict (or ``None``) of plain metadata for ``setup_context``. The trailing two slots are fixed (``_FWD_TRAILING_SLOTS``); everything before them is a user output. @@ -1367,6 +1528,7 @@ def register_custom_op_with_autograd( bwd_impl=bwd_impl, fwd_fake_impl=fwd_fake_impl, bwd_fake_impl=bwd_fake_impl, + fwd_tags=fwd_tags, ) except (ImportError, AttributeError, RuntimeError, TypeError) as e: record_compile_disabled( @@ -1386,6 +1548,7 @@ def _register_custom_op_with_autograd_impl( bwd_impl: Callable[[Any], Any], fwd_fake_impl: Callable[[Any], Tuple[Any, ...]], bwd_fake_impl: Callable[[Any], Tuple[Any, ...]], + fwd_tags: Sequence[torch.Tag], ) -> Callable[..., Any]: """Body of :func:`register_custom_op_with_autograd`; see it for semantics.""" # Existence check at the API boundary: every ``input_tensors_for_grad`` name @@ -1403,21 +1566,30 @@ def _register_custom_op_with_autograd_impl( impl=fwd_impl, fake_impl=fwd_fake_impl, pack_result=_pack_fwd_result, + tags=fwd_tags, ) bwd_qualname = f"{_TE_OP_NAMESPACE}::{op_name}_backward_base" num_grad_inputs = len(input_tensors_for_grad) + grad_targets = fwd_op.plan.resolve_grad_targets(input_tensors_for_grad) + fields = {field.name: field for field in fwd_op.plan.fields} + grad_types = ", ".join( + "Tensor[]" if fields[name].is_list else "Tensor" for name in input_tensors_for_grad + ) + if num_grad_inputs == 1: + grad_types = f"({grad_types})" # Preserve a one-element tuple in the schema. bwd_op = _register_op( name=f"{op_name}_backward", arg_type=bwd_arg_type, impl=bwd_impl, fake_impl=bwd_fake_impl, pack_result=lambda grads: _pack_bwd_result(grads, num_grad_inputs, bwd_qualname), + return_schema=f"({grad_types})", ) autograd_common = { "fwd_plan": fwd_op.plan, "bwd_plan": bwd_op.plan, - "grad_targets": fwd_op.plan.resolve_grad_targets(input_tensors_for_grad), + "grad_targets": grad_targets, "setup_context_user": setup_context, "fwd_fake_impl": fwd_fake_impl, } diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index 4cfe225952a..5ad9c8000b1 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -29,6 +29,7 @@ class TensorSpec: quantizer: Optional[Any] = None requires_grad: bool = False device: Optional[torch.device] = field(default=None) + with_gemm_swizzled_scales: Optional[bool] = None def __post_init__(self) -> None: # Own a private copy of the quantizer so usage changes (update_usage) @@ -37,6 +38,10 @@ def __post_init__(self) -> None: if self.quantizer is not None: q = self.quantizer self.quantizer = q.copy() if hasattr(q, "copy") else _copy.copy(q) + if self.with_gemm_swizzled_scales is None: + self.with_gemm_swizzled_scales = self.quantizer.storage_metadata(self.dtype)[ + "nontensor_kwargs" + ].get("with_gemm_swizzled_scales") @property def is_quantized(self) -> bool: @@ -94,9 +99,12 @@ def create_metadata(self) -> Dict[str, Any]: "dtype": self.dtype, "requires_grad": self.requires_grad, } - return self.quantizer.create_metadata( + meta = self.quantizer.create_metadata( tuple(self.shape), dtype=self.dtype, requires_grad=self.requires_grad ) + if self.with_gemm_swizzled_scales is not None: + meta["nontensor_kwargs"]["with_gemm_swizzled_scales"] = self.with_gemm_swizzled_scales + return meta def create_inner_tensors(self) -> List[torch.Tensor]: """Materialize the flat inner tensors (in :meth:`inner_names` order). @@ -145,6 +153,32 @@ def create_tensor(self) -> torch.Tensor: return self.assemble(self.create_inner_tensors()) +@dataclass +class GroupedTensorSpec(TensorSpec): + """Grouped buffers and metadata, including unquantized grouped parameters.""" + + metadata: Dict[str, Any] = field(default_factory=dict) + buffers: Dict[str, TensorSpec] = field(default_factory=dict) + + def inner_names(self) -> Tuple[str, ...]: + return tuple(self.buffers) + + def create_metadata(self) -> Dict[str, Any]: + return self.metadata + + def create_inner_tensors(self) -> List[torch.Tensor]: + return [spec.create_tensor() for spec in self.buffers.values()] + + def assemble(self, inner_tensors: List[torch.Tensor]) -> Any: + inner = dict(zip(self.inner_names(), inner_tensors)) + return self.metadata["cls"].__tensor_unflatten__( + inner, self.metadata, self.shape, make_contiguous_strides_for(self.shape) + ) + + def create_tensor(self) -> Any: + return self.assemble(self.create_inner_tensors()) + + def to_tensor_spec(tensor: Any) -> TensorSpec: """Build a :class:`TensorSpec` describing ``tensor``. @@ -152,7 +186,24 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: ``QuantizedTensor``. A *bare* storage exposes its (fake) dtype via ``_dtype`` rather than ``.dtype``. """ + from ..tensor.storage.grouped_tensor_storage import ( + GroupedTensorStorage, + ) # pylint: disable=import-outside-toplevel + requires_grad = bool(getattr(tensor, "requires_grad", False)) + if isinstance(tensor, GroupedTensorStorage): + names, metadata = GroupedTensorStorage.__tensor_flatten__(tensor) + buffers = {name: to_tensor_spec(getattr(tensor, name)) for name in names} + return GroupedTensorSpec( + shape=tuple(tensor.shape if isinstance(tensor, torch.Tensor) else tensor.logical_shape), + dtype=tensor.fake_dtype, + quantizer=tensor.quantizer, + requires_grad=requires_grad, + device=next(iter(buffers.values())).device, + with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales"), + metadata=metadata, + buffers=buffers, + ) dtype = getattr(tensor, "dtype", None) if dtype is None: dtype = getattr(tensor, "_dtype", None) @@ -162,4 +213,5 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: quantizer=getattr(tensor, "_quantizer", None), requires_grad=requires_grad, device=tensor.device, + with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales", None), ) diff --git a/transformer_engine/pytorch/module/_common.py b/transformer_engine/pytorch/module/_common.py index 3d04882698a..2c3dc3bf267 100644 --- a/transformer_engine/pytorch/module/_common.py +++ b/transformer_engine/pytorch/module/_common.py @@ -5,6 +5,7 @@ """Internal function used by multiple modules.""" import dataclasses +import math import queue from typing import Any, Callable, List, Optional, Tuple, Union @@ -13,10 +14,32 @@ from .. import cpp_extensions as tex from ..constants import TE_DType from ..distributed import in_fp8_activation_recompute_phase +from ..dynamo import TensorSpec from ..export import is_in_onnx_export_mode from ..quantization import FP8GlobalStateManager from ..tensor.hybrid_tensor import HybridQuantizer -from ..utils import get_default_init_method +from ..tensor.nvfp4_tensor import NVFP4Quantizer +from ..utils import get_default_init_method, get_device_compute_capability + + +def update_nvfp4_direct_output_spec(spec: TensorSpec) -> None: + """Match the NVFP4 layout emitted without a post-quantize swizzle.""" + quantizer = spec.quantizer + if not isinstance(quantizer, NVFP4Quantizer) or not quantizer.optimize_for_gemm: + return + rows, cols = math.prod(spec.shape[:-1]), spec.shape[-1] + if not (10, 0) <= get_device_compute_capability() <= (11, 0): + spec.with_gemm_swizzled_scales = False + elif quantizer.with_rht: + spec.with_gemm_swizzled_scales = bool(rows % 64 == 0 and cols % 128 == 0) + else: + spec.with_gemm_swizzled_scales = bool( + quantizer.with_2d_quantization + and not quantizer.row_scaled_nvfp4 + and not quantizer.nvfp4_use_4over6 + and rows % 128 == 0 + and cols % 128 == 0 + ) def sum_bias_grad(tensor: torch.Tensor) -> torch.Tensor: diff --git a/transformer_engine/pytorch/module/base.py b/transformer_engine/pytorch/module/base.py index 9ead1e73651..e8b8bc7409a 100644 --- a/transformer_engine/pytorch/module/base.py +++ b/transformer_engine/pytorch/module/base.py @@ -132,16 +132,29 @@ class UserBufferQuantizationMode(Enum): FP8 = "fp8" -def get_dummy_wgrad(shape: list, dtype: torch.dtype, zero=False) -> torch.Tensor: - """Returns a dummy tensor of given shape.""" +def get_dummy_wgrad( + shape: list, dtype: torch.dtype, zero: bool = False, zero_tensor: bool = False +) -> torch.Tensor: + """Returns a dummy tensor of given shape. + + With ``zero=True`` the cached dummy is filled with zeros. Otherwise, with + ``zero_tensor=True`` the dummy is a ZeroTensor: it has no storage and adding it costs + nothing, e.g. when autograd sums the dummy weight gradients of a weight used more than + once in one backward pass. + """ - key = (*shape, dtype) + # The dummy goes on the current device (the one the backward pass runs on), and the cache + # is keyed by it so that a dummy is never returned for a weight on another device. + device = torch.device("cuda", torch.cuda.current_device()) + if zero_tensor and not zero: + return torch._efficientzerotensor(shape, dtype=dtype, device=device) + key = (*shape, dtype, device) global _dummy_wgrads if key not in _dummy_wgrads: _dummy_wgrads[key] = torch.empty( shape, dtype=dtype, - device="cuda", + device=device, requires_grad=False, ) if zero: diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index ff3e01e2504..0c771d56c3b 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -5,8 +5,9 @@ """GroupedLinear API""" from dataclasses import dataclass -from typing import Any, Union, Optional, Callable, Tuple, List +from typing import Any, Union, Optional, Callable, Dict, Tuple, List from itertools import chain +import math import os import warnings import weakref @@ -45,6 +46,10 @@ requires_grad, resolve_grouped_linear_single_param_flags, get_nvtx_range_context, + warn_compile_eager_fallback, + warn_if_compile_disabled, + check_grouped_gemm_dims, + is_non_tn_fp8_gemm_supported, ) from ..distributed import ( set_tensor_model_parallel_attributes, @@ -62,6 +67,16 @@ general_grouped_gemm, general_grouped_gemm_for_grouped_tensor, ) +from ..cpp_extensions.gemm import get_cublas_workspace +from ..dynamo import ( + TensorSpec, + register_custom_op_with_autograd, + is_value_opaque_quantizer, +) +from ..dynamo.tensor_spec import GroupedTensorSpec, to_tensor_spec +from ..dynamo.custom_op import SavedTensorRef +from .linear import _fake_workspace_valid +from ._common import update_nvfp4_direct_output_spec from ..constants import GemmParallelModes, dist_group_type from ..jit import no_torch_dynamo from ..cpu_offload import is_cpu_offload_enabled, mark_not_offload, start_offload @@ -74,6 +89,7 @@ HybridQuantizer, IdentityQuantizer, MXFP8Quantizer, + NVFP4Quantizer, ) from ..quantized_tensor import ( QuantizedTensorStorage, @@ -125,7 +141,7 @@ def is_module_grouped_tensor_path_supported( device_capability = get_device_compute_capability() if not (9, 0) <= device_capability <= (11, 0): return False - cublaslt_version = tex.get_cublasLt_version() + cublaslt_version = _get_cublaslt_version() if cublaslt_version < 130300: return False if device_capability < (10, 0) and cublaslt_version < 130400: @@ -154,13 +170,20 @@ def is_module_grouped_tensor_path_supported( return False +@functools.lru_cache(maxsize=None) +@torch.compiler.assume_constant_result +def _get_cublaslt_version() -> int: + """Cached, Dynamo-constant cuBLASLt version (the pybind call is untraceable).""" + return tex.get_cublasLt_version() + + @dataclass(slots=True) class GroupedLinearFwdArgs: """Forward configuration and operands.""" inp: torch.Tensor weights: List[Union[torch.Tensor, QuantizedTensorStorage]] - biases: List[torch.Tensor] + biases: List[Union[torch.Tensor, QuantizedTensorStorage]] weight_workspaces: List[Union[torch.Tensor, QuantizedTensorStorage]] out: Optional[torch.Tensor] @@ -170,6 +193,7 @@ class GroupedLinearFwdArgs: input_requires_grad: bool weights_requires_grad: bool + bias_requires_grad: bool input_quantizers: List[Quantizer] weight_quantizers: List[Quantizer] @@ -186,9 +210,7 @@ class GroupedLinearFwdArgs: fp8_calibration: bool save_original_input: bool backward_override: Optional[str] - fprop_use_split_accumulator: bool - dgrad_use_split_accumulator: bool - wgrad_use_split_accumulator: bool + fp8_recipe: Optional[Recipe] debug: bool is_first_microbatch: Optional[bool] @@ -203,6 +225,49 @@ class GroupedLinearFwdArgs: wgrad_store: Optional[Any] cpu_offloading: bool is_grad_enabled: bool + compiled_op: bool = False + fprop_use_split_accumulator: bool = _2X_ACC_FPROP + dgrad_use_split_accumulator: bool = _2X_ACC_DGRAD + wgrad_use_split_accumulator: bool = _2X_ACC_WGRAD + + def compile_unsupported_reason(self) -> Optional[str]: + """Reason this config can't use the torch.compile custom-op path (else None).""" + if self.debug: + return "debug instrumentation (nvidia-dlfw-inspect)" + if is_distributed_weight(self.weights[0]): + return "a DistributedWeight (custom weight parallelism, e.g. GTP)" + if self.out is not None or self.dgrad_out is not None: + return "a user-provided out/dgrad_out buffer (the op would return an input alias)" + if self.cpu_offloading: + return "CPU activation offloading" + if self.wgrad_store is not None: + return "delayed wgrad compute (wgrad_store)" + if self.fuse_wgrad_accumulation: + return "fuse_wgrad_accumulation (main_grad)" + if self.fp8_calibration: + return "fp8_calibration" + if self.use_grouped_tensor_path: + if self.fp8 and not isinstance( + self.input_quantizers[0], + (Float8CurrentScalingQuantizer, MXFP8Quantizer, NVFP4Quantizer), + ): + return "a fused-path recipe other than current scaling, MXFP8, or NVFP4" + elif self.m_splits is None: + return "m_splits passed as a device tensor inside the compiled region" + if any(w.requires_grad != self.weights_requires_grad for w in self.weights): + return "mixed requires_grad across the weights list" + for quantizer_list in ( + self.input_quantizers, + self.weight_quantizers, + self.output_quantizers, + self.grad_input_quantizers, + self.grad_weight_quantizers, + self.grad_output_quantizers, + ): + for quantizer in quantizer_list: + if quantizer is not None and not is_value_opaque_quantizer(quantizer): + return "a quantizer not registered as a torch.compile value-opaque type" + return None @dataclass(slots=True) @@ -210,6 +275,7 @@ class GroupedLinearBwdArgs: """Backward configuration and saved operands.""" grad_output: Optional[torch.Tensor] = None + inputmat_full: Optional[torch.Tensor] = None inputmats: List[Union[torch.Tensor, QuantizedTensorStorage]] = None weights_fp8: List[Union[torch.Tensor, QuantizedTensorStorage]] = None saved_weights: List[Union[torch.Tensor, QuantizedTensorStorage]] = None @@ -233,8 +299,7 @@ class GroupedLinearBwdArgs: activation_dtype: Optional[torch.dtype] = None fp8: bool = False backward_override: Optional[str] = None - dgrad_use_split_accumulator: bool = _2X_ACC_DGRAD - wgrad_use_split_accumulator: bool = _2X_ACC_WGRAD + fp8_recipe: Optional[Recipe] = None save_original_input: bool = False debug: bool = False @@ -248,28 +313,69 @@ class GroupedLinearBwdArgs: reduce_and_update_bwd_fp8_tensors: bool = False cpu_offloading: bool = False + compiled_op: bool = False + dgrad_use_split_accumulator: bool = _2X_ACC_DGRAD + wgrad_use_split_accumulator: bool = _2X_ACC_WGRAD def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" saved = restore_from_func_ctx(ctx) n = self.num_gemms - self.inputmats = list(saved[:n]) - self.weights_fp8 = list(saved[n : 2 * n]) - self.saved_weights = list(saved[2 * n : 3 * n]) - self.biases = list(saved[3 * n : 4 * n]) + self.inputmat_full = saved[0] + self.inputmats = saved[1 : 1 + n] + self.weights_fp8 = saved[1 + n : 1 + 2 * n] + self.saved_weights = saved[1 + 2 * n : 1 + 3 * n] + self.biases = saved[1 + 3 * n : 1 + 4 * n] + + +def _grouped_linear_configure_quantizers(args: GroupedLinearFwdArgs, weights) -> None: + """Configure operand usage after materializing weights.""" + if args.input_quantizers[0] is not None: + for input_quantizer in args.input_quantizers: + input_quantizer.set_usage( + rowwise=True, + columnwise=( + args.is_grad_enabled + and args.weights_requires_grad + and not args.save_original_input + and args.backward_override is None + ), + ) + columnwise_usage = args.is_grad_enabled and args.input_requires_grad + if args.backward_override is not None: + columnwise_usage = False + if not columnwise_usage: + columnwise_usage = ( + is_fp8_activation_recompute_enabled() and not in_fp8_activation_recompute_phase() + ) + if args.weight_quantizers[0] is not None and ( + not isinstance(weights[0], QuantizedTensorStorage) or args.debug + ): + for weight_quantizer in args.weight_quantizers: + weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) + elif isinstance(weights[0], QuantizedTensorStorage): + args.weight_quantizers = [weight._quantizer for weight in weights] + if args.output_quantizers[0] is not None: + for output_quantizer in args.output_quantizers: + output_quantizer.set_usage(rowwise=True, columnwise=False) def _grouped_linear_forward_impl( args: GroupedLinearFwdArgs, ) -> Tuple[Any, ...]: - """Compute the legacy forward and collect tensors needed by backward.""" + """Forward implementation for the grouped linear layer (legacy, non-fused path). + + Returns ``(out, *new_workspaces, tensors_to_save, ctx_attrs)``. + ``new_workspaces`` are the freshly produced FP8 weight workspaces (returned + alongside ``out`` so the caller can refresh its cache). The trailing two are + ``None`` when gradients are disabled. + """ inp = args.inp - weights = list(args.weights) - biases = list(args.biases) + weights = args.weights + biases = args.biases num_gemms = args.num_gemms - m_splits = list(args.m_splits) + m_splits = args.m_splits input_quantizers = args.input_quantizers - weight_quantizers = args.weight_quantizers output_quantizers = args.output_quantizers activation_dtype = args.activation_dtype fp8 = args.fp8 @@ -286,38 +392,8 @@ def _grouped_linear_forward_impl( if is_dist_weight: weights = materialize_weight_for_forward(weights) - # Configure quantizers - if input_quantizers[0] is not None: - for input_quantizer in input_quantizers: - input_quantizer.set_usage( - rowwise=True, - columnwise=( - is_grad_enabled - and weight_requires_grad - and not save_original_input - and backward_override is None - ), - ) - columnwise_usage = is_grad_enabled and args.input_requires_grad - if backward_override is not None: - columnwise_usage = False - if not columnwise_usage: - columnwise_usage = ( - is_fp8_activation_recompute_enabled() and not in_fp8_activation_recompute_phase() - ) - # No need to set the quantizer states if weight is already quantized - # for debug mode we create quantizer every iteration, thus we need to set the quantizer states - if weight_quantizers[0] is not None and ( - not isinstance(weights[0], QuantizedTensorStorage) or debug - ): - for weight_quantizer in weight_quantizers: - weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) - elif isinstance(weights[0], QuantizedTensorStorage): - # If weights are already quantized, no need to set quantizer states - weight_quantizers = [weight._quantizer for weight in weights] - if output_quantizers[0] is not None: - for output_quantizer in output_quantizers: - output_quantizer.set_usage(rowwise=True, columnwise=False) + _grouped_linear_configure_quantizers(args, weights) + weight_quantizers = args.weight_quantizers # Initialize input tensors in_features = weights[0].size(-1) @@ -328,14 +404,30 @@ def _grouped_linear_forward_impl( ) inp_view = inp.reshape(-1, in_features) - inputmats, _ = _split_quantization._split_quantize( - inp_view, - m_splits, - input_quantizers, - activation_dtype, - with_quantized_output=fp8 or debug, - disable_bulk_allocation=cpu_offloading, - ) + fp8_or_debug = fp8 or debug + inputmat_full = None + if fp8_or_debug: + # Disable bulk allocation when CPU offloading is active: offloading skips small + # tensors (like scales), but bulk allocation shares storage across all tensors, + # so if scales can't be offloaded, nothing in the group can be offloaded. + # The compiled op also disables it: bulk-allocated storages alias each + # other, and the op returns them as saved tensors. + inputmats, _ = _split_quantization._split_quantize( + inp_view, + m_splits, + with_quantized_output=True, + quantizers=input_quantizers, + activation_dtype=activation_dtype, + disable_bulk_allocation=cpu_offloading or args.compiled_op, + ) + else: + # Plain path: split views of one (possibly cast) buffer. Save that + # buffer once (backward re-splits it) instead of N aliasing views -- + # except under CPU offloading, which marks the individual views. + inputmat_full = cast_if_needed(inp_view, activation_dtype) + inputmats = torch.split(inputmat_full, m_splits) + if cpu_offloading: + inputmat_full = None if cpu_offloading: start_offload(*inputmats) @@ -343,7 +435,7 @@ def _grouped_linear_forward_impl( # Initialize weights weights_fp8: list new_workspaces = [None] * num_gemms - if fp8 or debug: + if fp8_or_debug: weights_fp8 = [] update_ws = args.is_first_microbatch is None or args.is_first_microbatch for i in range(num_gemms): @@ -375,6 +467,9 @@ def _grouped_linear_forward_impl( ) # Perform GEMM + fprop_use_split_accumulator = args.fprop_use_split_accumulator + if fp8 and hasattr(args.fp8_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = args.fp8_recipe.fp8_gemm_fprop.use_split_accumulator general_grouped_gemm( weights_fp8, inputmats, @@ -385,7 +480,7 @@ def _grouped_linear_forward_impl( m_splits=m_splits, bias=biases, use_bias=use_bias, - use_split_accumulator=args.fprop_use_split_accumulator, + use_split_accumulator=fprop_use_split_accumulator, ) if args.fp8_calibration: @@ -401,8 +496,8 @@ def _grouped_linear_forward_impl( # TODO: update after #1638 is merged. # pylint: disable=fixme if weight_requires_grad: if save_original_input: + inputmat_full = SavedTensorRef("inp") if args.compiled_op else inp inputmats = [None] * num_gemms - inputmats[0] = inp else: for inputmat in inputmats: if isinstance(inputmat, QuantizedTensorStorage): @@ -412,39 +507,334 @@ def _grouped_linear_forward_impl( inputmat.update_usage(rowwise_usage=True, columnwise_usage=False) else: inputmat.update_usage(rowwise_usage=False, columnwise_usage=True) + if inputmat_full is not None: + # Plain path: the views in ``inputmats`` are not saved. + inputmats = [None] * num_gemms + if inputmat_full is inp_view: + # No-op cast: inp_view is inp itself or a view of it. + inputmat_full = SavedTensorRef("inp") if args.compiled_op else inp else: + inputmat_full = None inputmats = [None] * num_gemms # Original weights are only needed by high_precision dgrad. The weakrefs # used for fused wgrad accumulation serve a different purpose: restoring # Python parameter attributes without keeping the parameter alive here. - saved_weights = ( - weights - if backward_override == "high_precision" and args.input_requires_grad - else [None] * num_gemms - ) + save_origin_weights = backward_override == "high_precision" and args.input_requires_grad + saved_weights = args.weights if save_origin_weights else [None] * num_gemms if is_dist_weight: - # GTP: gathered workspace is transient (re-gathered in backward), don't save it. weights_fp8 = [None] * num_gemms saved_weights = args.weights - tensors_to_save = (*inputmats, *weights_fp8, *saved_weights, *biases) + if args.compiled_op: + weights_fp8 = _saved_grouped_weights(args, new_workspaces, weights_fp8) + if save_origin_weights: + saved_weights = [SavedTensorRef("weights", i) for i in range(num_gemms)] + biases = [ + SavedTensorRef("biases", i) if bias is not None and bias is args.biases[i] else bias + for i, bias in enumerate(biases) + ] - return out.view(-1, *inp.shape[1:-1], out.shape[-1]), new_workspaces, tensors_to_save + tensors_to_save = ( + inputmat_full, + *inputmats, + *weights_fp8, + *saved_weights, + *biases, + ) + + # [*, in_features] -> [*, out_features] + out = out.view(-1, *inp.shape[1:-1], out.shape[-1]) + return (out, *new_workspaces, tensors_to_save, None) + + +def _fake_single_grouped_weight(weight, quantizer, dtype, num_gemms, cache_weight): + """Describe the grouped weight allocation made by the native quantizer.""" + shape = tuple(weight.metadata["nontensor_kwargs"]["shape"]) + total = math.prod(shape) + metadata = dict(weight.metadata) + kwargs = dict(metadata["nontensor_kwargs"]) + kwargs.update(dtype=dtype, quantizer=quantizer) + metadata.update(nontensor_kwargs=kwargs, requires_grad=False) + + def spec(size, buffer_dtype): + return TensorSpec(shape=(size,), dtype=buffer_dtype, device=weight.device) + + if quantizer is None: + metadata.update(cls=GroupedTensorStorage, is_tensor=False) + buffers = {"rowwise_data": spec(total, dtype)} + else: + metadata.update( + cls=GroupedTensor if cache_weight or not quantizer.internal else GroupedTensorStorage, + is_tensor=cache_weight or not quantizer.internal, + ) + kwargs.update( + shapes=None, offsets=None, scale_inv_offsets=None, columnwise_scale_inv_offsets=None + ) + buffers = {} + if isinstance(quantizer, Float8CurrentScalingQuantizer): + non_tn = is_non_tn_fp8_gemm_supported() + if quantizer.rowwise_usage or non_tn: + buffers["rowwise_data"] = spec(total, torch.uint8) + buffers["scale_inv"] = spec(num_gemms, torch.float32) + if quantizer.columnwise_usage and not non_tn: + buffers["columnwise_data"] = spec(total, torch.uint8) + buffers["columnwise_scale_inv"] = spec(num_gemms, torch.float32) + buffers["amax"] = spec(num_gemms, torch.float32) + buffers["scale"] = spec(num_gemms, torch.float32) + kwargs["with_gemm_swizzled_scales"] = False + else: + for columnwise, usage in ( + (False, quantizer.rowwise_usage), + (True, quantizer.columnwise_usage), + ): + if usage: + prefix = "columnwise_" if columnwise else "" + buffers["columnwise_data" if columnwise else "rowwise_data"] = spec( + total, torch.uint8 + ) + buffers[prefix + "scale_inv"] = spec( + math.prod(quantizer.get_scale_shape(shape, columnwise)), torch.uint8 + ) + kwargs["with_gemm_swizzled_scales"] = quantizer.optimize_for_gemm + metadata["shared_scale"] = ( + "scale_inv" in buffers + and isinstance(quantizer, Float8CurrentScalingQuantizer) + and "columnwise_scale_inv" in buffers + ) + if metadata["shared_scale"]: + del buffers["columnwise_scale_inv"] + buffers = { + name: buffers[name] for name, _ in GroupedTensorStorage._INNER_TENSORS if name in buffers + } + outer_shape = (num_gemms, *weight.shape[-2:]) if kwargs["shapes"] is not None else shape + return GroupedTensorSpec( + shape=outer_shape, + dtype=dtype, + quantizer=quantizer, + device=weight.device, + with_gemm_swizzled_scales=kwargs["with_gemm_swizzled_scales"], + metadata=metadata, + buffers=buffers, + ) + + +def _fake_grouped_weights(args: GroupedLinearFwdArgs, weight_quantizers): + """Describe weight casts, quantization and workspace aliases for either path.""" + new_workspaces = [None] * len(args.weights) + saved_weights = [None] * len(args.weights) + for i, weight in enumerate(args.weights): + if (args.fp8 and weight.is_quantized) or ( + not args.fp8 and weight.dtype == args.activation_dtype + ): + saved_weights[i] = weight + continue + quantizer = weight_quantizers[i] if args.fp8 else None + workspace = args.weight_workspaces[i] if args.fp8 else None + if args.single_grouped_weight: + workspace_valid = workspace is not None and ( + workspace.metadata["nontensor_kwargs"]["quantizer"] == quantizer + and ( + not quantizer.columnwise_usage + or ( + isinstance(quantizer, Float8CurrentScalingQuantizer) + and is_non_tn_fp8_gemm_supported() + ) + or "columnwise_data" in workspace.buffers + ) + ) + else: + workspace_valid = workspace is not None and _fake_workspace_valid(workspace, quantizer) + if workspace_valid: + saved_weights[i] = workspace + continue + if args.single_grouped_weight: + weightmat = _fake_single_grouped_weight( + weight, quantizer, args.activation_dtype, args.num_gemms, args.cache_weight + ) + else: + weightmat = TensorSpec( + shape=tuple(weight.shape), + dtype=args.activation_dtype, + quantizer=quantizer, + device=weight.device, + ) + if args.fp8 and args.cache_weight: + if weightmat.quantizer is not None: + weightmat.quantizer.internal = False + new_workspaces[i] = weightmat + saved_weights[i] = weightmat + return new_workspaces, _saved_grouped_weights(args, new_workspaces, saved_weights) + + +def _grouped_linear_forward_fake( + args: GroupedLinearFwdArgs, +) -> Tuple[Any, ...]: + """Shape/metadata-only twin of :func:`_grouped_linear_forward_impl` for + torch.compile. Only mirrors configs the compiled path admits (see + ``compile_unsupported_reason``): no debug, offloading, distributed weights, + calibration, or fused GroupedTensor path. + """ + inp = args.inp + weights = list(args.weights) + num_gemms = args.num_gemms + m_splits = list(args.m_splits) + input_quantizers = args.input_quantizers + weight_quantizers = args.weight_quantizers + output_quantizers = args.output_quantizers + activation_dtype = args.activation_dtype + fp8 = args.fp8 + is_grad_enabled = args.is_grad_enabled + save_original_input = args.save_original_input + backward_override = args.backward_override + weight_requires_grad = args.weights_requires_grad + in_features = weights[0].shape[-1] + out_features = weights[0].shape[0] + + # Mirror the impl's quantizer usage setup exactly (buffer layouts must agree). + if input_quantizers[0] is not None: + for input_quantizer in input_quantizers: + input_quantizer.set_usage( + rowwise=True, + columnwise=( + is_grad_enabled + and weight_requires_grad + and not save_original_input + and backward_override is None + ), + ) + columnwise_usage = is_grad_enabled and args.input_requires_grad + if backward_override is not None: + columnwise_usage = False + if not columnwise_usage: + columnwise_usage = ( + is_fp8_activation_recompute_enabled() and not in_fp8_activation_recompute_phase() + ) + if weight_quantizers[0] is not None and not weights[0].is_quantized: + for weight_quantizer in weight_quantizers: + weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) + elif weights[0].is_quantized: + weight_quantizers = [weight.quantizer for weight in weights] + if output_quantizers[0] is not None: + for output_quantizer in output_quantizers: + output_quantizer.set_usage(rowwise=True, columnwise=False) + + # Input pipeline: quantized per-split storages (fp8) or one full cast buffer. + inputmat_full = None + inputmats: List[Optional[TensorSpec]] = [None] * num_gemms + inputmat_full_aliases_inp = False + if fp8: + inputmats = [ + TensorSpec( + shape=(m_splits[i], in_features), + dtype=activation_dtype, + quantizer=input_quantizers[i], + device=inp.device, + ) + for i in range(num_gemms) + ] + for inputmat in inputmats: + update_nvfp4_direct_output_spec(inputmat) + else: + inputmat_full_aliases_inp = inp.dtype == activation_dtype + inputmat_full = TensorSpec( + shape=(sum(m_splits), in_features), + dtype=activation_dtype, + device=inp.device, + ) + + new_workspaces, weights_fp8 = _fake_grouped_weights(args, weight_quantizers) + + # Bias pipeline. + bias_dtype = activation_dtype + if fp8 and activation_dtype == torch.float32: + bias_dtype = torch.bfloat16 + saved_biases = [None] * num_gemms + for i in range(num_gemms): + bias = args.biases[i] + if bias is None: + continue + if not args.use_bias or bias.dtype == bias_dtype: + saved_biases[i] = SavedTensorRef("biases", i) + else: + saved_biases[i] = TensorSpec( + shape=tuple(bias.shape), dtype=bias_dtype, device=bias.device + ) + + out = TensorSpec( + shape=(*tuple(inp.shape[:-1]), out_features), + dtype=activation_dtype, + quantizer=None, + requires_grad=is_grad_enabled + and (args.input_requires_grad or weight_requires_grad or args.bias_requires_grad), + device=inp.device, + ) + + tensors_to_save = None + if is_grad_enabled: + if weight_requires_grad: + if save_original_input: + inputmat_full = SavedTensorRef("inp") + inputmats = [None] * num_gemms + else: + if fp8: + for inputmat in inputmats: + if backward_override is not None: + inputmat.update_usage(rowwise_usage=True, columnwise_usage=False) + else: + inputmat.update_usage(rowwise_usage=False, columnwise_usage=True) + elif inputmat_full_aliases_inp: + inputmat_full = SavedTensorRef("inp") + else: + inputmat_full = None + inputmats = [None] * num_gemms + + save_origin_weights = backward_override == "high_precision" and args.input_requires_grad + saved_weights = [ + SavedTensorRef("weights", i) if save_origin_weights else None for i in range(num_gemms) + ] + + tensors_to_save = ( + inputmat_full, + *inputmats, + *weights_fp8, + *saved_weights, + *saved_biases, + ) + + return (out, *new_workspaces, tensors_to_save, None) + + +def _saved_grouped_weights(args, new_workspaces, weights): + """Return fresh weights once and refer to existing inputs or cache outputs.""" + saved = [] + for i, weight in enumerate(weights): + if weight is None: + saved.append(None) + elif weight is args.weights[i]: + saved.append(SavedTensorRef("weights", i)) + elif weight is new_workspaces[i]: + saved.append(SavedTensorRef(None, 1 + i)) + elif args.weight_workspaces and weight is args.weight_workspaces[i]: + saved.append(SavedTensorRef("weight_workspaces", i)) + else: + saved.append(weight) + return saved def _grouped_linear_setup_ctx( bwd_args: GroupedLinearBwdArgs, fwd_args: GroupedLinearFwdArgs, -) -> None: - """Populate backward arguments from the forward configuration.""" + fwd_outputs: Tuple[Any, ...], + ctx_attrs: Dict, + tensors_to_save_from_forward: Tuple[Any, ...], +) -> Tuple[Any, ...]: + """Populate backward configuration and pass through its saved tensors.""" + del fwd_outputs, ctx_attrs num_gemms = fwd_args.num_gemms - weights = fwd_args.weights - weight_quantizers = fwd_args.weight_quantizers - if isinstance(weights[0], QuantizedTensorStorage) and not fwd_args.debug: - weight_quantizers = [weight._quantizer for weight in weights] + weights = fwd_args.weights bwd_args.input_quantizers = fwd_args.input_quantizers - bwd_args.weight_quantizers = weight_quantizers + bwd_args.weight_quantizers = fwd_args.weight_quantizers bwd_args.grad_input_quantizers = fwd_args.grad_input_quantizers bwd_args.grad_weight_quantizers = fwd_args.grad_weight_quantizers bwd_args.grad_output_quantizers = fwd_args.grad_output_quantizers @@ -460,6 +850,7 @@ def _grouped_linear_setup_ctx( bwd_args.activation_dtype = fwd_args.activation_dtype bwd_args.fp8 = fwd_args.fp8 bwd_args.backward_override = fwd_args.backward_override + bwd_args.fp8_recipe = fwd_args.fp8_recipe bwd_args.dgrad_use_split_accumulator = fwd_args.dgrad_use_split_accumulator bwd_args.wgrad_use_split_accumulator = fwd_args.wgrad_use_split_accumulator bwd_args.save_original_input = fwd_args.save_original_input @@ -470,6 +861,7 @@ def _grouped_linear_setup_ctx( bwd_args.wgrad_store = fwd_args.wgrad_store bwd_args.cpu_offloading = fwd_args.cpu_offloading bwd_args.dgrad_out = fwd_args.dgrad_out + bwd_args.compiled_op = fwd_args.compiled_op if fwd_args.fuse_wgrad_accumulation and fwd_args.weights_requires_grad: # Keep weakrefs to weights to preserve attributes like main_grad @@ -493,6 +885,8 @@ def _grouped_linear_setup_ctx( bwd_args.grad_weight_quantizers = [None] * num_gemms bwd_args.grad_output_quantizers = [None] * num_gemms + return tensors_to_save_from_forward + def _finish_wgrad(weight, main_grad, wgrad, fuse_wgrad_accumulation): """Return the gradient expected by autograd or MCore's main-grad hooks.""" @@ -502,7 +896,10 @@ def _finish_wgrad(weight, main_grad, wgrad, fuse_wgrad_accumulation): return None weight.grad_added_to_main_grad = True return get_dummy_wgrad( - list(main_grad.shape), weight.dtype, zero=getattr(weight, "zero_out_wgrad", False) + list(main_grad.shape), + weight.dtype, + zero=getattr(weight, "zero_out_wgrad", False), + zero_tensor=getattr(weight, "zero_tensor_dummy_wgrad", False), ) @@ -517,15 +914,19 @@ def _grouped_linear_backward_impl( """ grad_output = args.grad_output num_gemms = args.num_gemms - m_splits = list(args.m_splits) - inputmats = list(args.inputmats) - weights = list(args.weights_fp8) - saved_weights = list(args.saved_weights) - biases = list(args.biases) + m_splits = args.m_splits + inputmats = args.inputmats + weights = args.weights_fp8 + saved_weights = args.saved_weights + biases = args.biases device = grad_output.device in_features = args.weights_shape_1 dgrad = None + # Plain (non-quantized) inputs are saved as one full buffer; re-split it. + if args.inputmat_full is not None and not args.save_original_input: + inputmats = list(torch.split(args.inputmat_full.reshape(-1, in_features), m_splits)) + # Restore from weakrefs to get original weight python objects # (preserves attributes like main_grad, grad_added_to_main_grad, etc.) # Only needed when fuse_wgrad_accumulation is enabled. @@ -549,8 +950,8 @@ def _grouped_linear_backward_impl( origin_weight.main_grad = main_grad # Preprocess grad output - grad_output_view = grad_output.contiguous().view(-1, grad_output.shape[-1]) - out_features = grad_output_view.shape[-1] + out_features = grad_output.shape[-1] + grad_output_view = grad_output.contiguous().view(-1, out_features) grad_output_reference = args.grad_output_quantizers[0] if args.fp8 and isinstance(grad_output_reference, HybridQuantizer): # Usage is a runtime decision, not part of generation validation. @@ -567,8 +968,12 @@ def _grouped_linear_backward_impl( args.grad_output_quantizers, args.activation_dtype, with_quantized_output=args.fp8 or args.debug, - compute_dbias=(args.fp8 or args.debug) and (args.use_bias or args.debug), - disable_bulk_allocation=args.cpu_offloading, + compute_dbias=(args.fp8 or args.debug) and args.use_bias, + disable_bulk_allocation=( + args.cpu_offloading + and isinstance(grad_output_reference, HybridQuantizer) + and not _split_quantization._uses_identity_quantizer(grad_output_reference) + ), ) if grad_biases is None: grad_biases = [None] * num_gemms @@ -586,6 +991,9 @@ def _grouped_linear_backward_impl( weights = materialize_weight_for_backward(origin_weights) if args.requires_dgrad: + dgrad_use_split_accumulator = args.dgrad_use_split_accumulator + if (args.fp8 or args.debug) and hasattr(args.fp8_recipe, "fp8_gemm_dgrad"): + dgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_dgrad.use_split_accumulator dgrad = _GroupedLinear._validate_or_alloc_output( args.dgrad_out, sum(m_splits), @@ -627,7 +1035,7 @@ def _grouped_linear_backward_impl( layout="NN", m_splits=m_splits, grad=True, - use_split_accumulator=args.dgrad_use_split_accumulator, + use_split_accumulator=dgrad_use_split_accumulator, ) if args.weights_requires_grad: @@ -639,8 +1047,21 @@ def _grouped_linear_backward_impl( raise RuntimeError( "distributed-weight GroupedLinear requires delay_wgrad_compute=False." ) + wgrad_use_split_accumulator = args.wgrad_use_split_accumulator + if args.fp8 and hasattr(args.fp8_recipe, "fp8_gemm_wgrad"): + wgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_wgrad.use_split_accumulator if args.fuse_wgrad_accumulation: wgrad_list = main_grads + elif args.compiled_op: + # Packed allocation would make the returned wgrads alias each other. + wgrad_list = [ + torch.empty( + (out_features, in_features), + dtype=args.activation_dtype, + device=device, + ) + for _ in range(num_gemms) + ] else: wgrad_packed = torch.empty( num_gemms, @@ -655,7 +1076,7 @@ def _grouped_linear_backward_impl( del weights if args.save_original_input: - inp = inputmats[0] + inp = args.inputmat_full inp_view = inp.reshape(-1, in_features) if args.input_quantizers[0] is not None: for input_quantizer in args.input_quantizers: @@ -688,7 +1109,7 @@ def _grouped_linear_backward_impl( m_splits=m_splits, use_bias=args.use_bias if grad_biases[0] is None else None, bias=biases, - use_split_accumulator=args.wgrad_use_split_accumulator, + use_split_accumulator=wgrad_use_split_accumulator, accumulate=( accumulate_wgrad_into_param_main_grad if not is_dist_weight and not args.origin_weights_overwrite_main_grad @@ -706,7 +1127,10 @@ def _grouped_linear_backward_impl( grad_biases[i] = grad_biases_[i] del grad_biases_ - clear_tensor_data(*inputmats) + # Deallocate input tensor (in-place storage resize: not allowed on + # the compiled path, where the inputs belong to the op caller). + if not args.compiled_op: + clear_tensor_data(*inputmats) if is_dist_weight: wgrad_list = finalize_weight_grads(origin_weights, wgrad_list) @@ -730,12 +1154,90 @@ def _grouped_linear_backward_impl( return (dgrad_out, wgrad_list, list(grad_biases)) +def _fake_grouped_grads(args, in_features, compute_bias): + """Describe gradient allocations shared by both compiled paths.""" + grad_output = args.grad_output + out_features = grad_output.shape[-1] + dgrad = None + if args.requires_dgrad: + dgrad = TensorSpec( + shape=(*tuple(grad_output.shape[:-1]), in_features), + dtype=args.activation_dtype, + device=grad_output.device, + ) + + single_weight = getattr(args, "single_grouped_weight", False) + single_bias = getattr(args, "single_grouped_bias", False) + num_weights = 1 if single_weight else args.num_gemms + num_biases = 1 if single_bias else args.num_gemms + wgrad_list: List[Optional[TensorSpec]] = [None] * num_weights + if args.weights_requires_grad and not args.fuse_wgrad_accumulation: + wgrad_list = [ + TensorSpec( + shape=( + (args.num_gemms, out_features, in_features) + if single_weight + else (out_features, in_features) + ), + dtype=args.activation_dtype, + device=grad_output.device, + ) + for _ in range(num_weights) + ] + + grad_biases: List[Optional[TensorSpec]] = [None] * num_biases + if compute_bias: + grad_biases = [ + TensorSpec( + shape=(args.num_gemms, out_features) if single_bias else (out_features,), + dtype=args.activation_dtype, + device=grad_output.device, + ) + for _ in range(num_biases) + ] + + return (dgrad, wgrad_list, grad_biases) + + +def _grouped_linear_backward_fake( + args: GroupedLinearBwdArgs, +) -> Tuple[Optional[TensorSpec], List[Optional[TensorSpec]], List[Optional[TensorSpec]]]: + """Allocation-free fake of :func:`_grouped_linear_backward_impl` on ``TensorSpec``.""" + + # Mirror the impl's hybrid grad-output usage retarget. + grad_output_reference = args.grad_output_quantizers[0] + if args.fp8 and isinstance(grad_output_reference, HybridQuantizer): + for grad_output_quantizer in args.grad_output_quantizers: + grad_output_quantizer.set_usage( + rowwise=args.requires_dgrad, + columnwise=args.weights_requires_grad, + ) + + return _fake_grouped_grads( + args, args.weights_shape_1, args.use_bias and (args.weights_requires_grad or args.fp8) + ) + + +# Custom op used under ``torch.compile``. +_grouped_linear_op = register_custom_op_with_autograd( + op_name="grouped_linear", + input_tensors_for_grad=["inp", "weights", "biases"], + fwd_arg_type=GroupedLinearFwdArgs, + fwd_impl=_grouped_linear_forward_impl, + fwd_fake_impl=_grouped_linear_forward_fake, + setup_context=_grouped_linear_setup_ctx, + bwd_arg_type=GroupedLinearBwdArgs, + bwd_impl=_grouped_linear_backward_impl, + bwd_fake_impl=_grouped_linear_backward_fake, +) + + @dataclass(slots=True) -class GroupedLinearFusedBwdArgs: +class GroupedLinearGraphSafeBwdArgs: """Backward configuration and saved grouped operands.""" grad_output: Optional[torch.Tensor] = None - inputmat: Any = None + inputmat: Union[torch.Tensor, QuantizedTensorStorage] = None weights_fp8: List[Union[torch.Tensor, QuantizedTensorStorage]] = None m_splits_tensor: Optional[torch.Tensor] = None base_split_offsets: Optional[torch.Tensor] = None @@ -758,8 +1260,7 @@ class GroupedLinearFusedBwdArgs: single_grouped_weight: bool = False single_grouped_bias: bool = False is_first_microbatch: Optional[bool] = None - dgrad_use_split_accumulator: bool = _2X_ACC_DGRAD - wgrad_use_split_accumulator: bool = _2X_ACC_WGRAD + fp8_recipe: Optional[Recipe] = None fuse_wgrad_accumulation: bool = False origin_weight_refs: Optional[Any] = None origin_weights_overwrite_main_grad: bool = False @@ -767,12 +1268,18 @@ class GroupedLinearFusedBwdArgs: wgrad_store: Optional[WeightGradStore] = None reduce_and_update_bwd_fp8_tensors: bool = False + compiled_op: bool = False + dgrad_use_split_accumulator: bool = _2X_ACC_DGRAD + wgrad_use_split_accumulator: bool = _2X_ACC_WGRAD + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Restore the grouped operands and split metadata.""" saved = restore_from_func_ctx(ctx) n = 1 if self.single_grouped_weight else self.num_gemms self.inputmat = saved[0] - self.weights_fp8 = list(saved[1 : 1 + n]) + self.weights_fp8 = saved[1 : 1 + n] + if self.compiled_op: + return ( self.m_splits_tensor, self.base_split_offsets, @@ -781,14 +1288,13 @@ def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: ) = saved[1 + n :] -def _grouped_linear_fused_forward(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: +def _grouped_linear_graph_safe_forward(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: """Compute the grouped forward and collect tensors needed by backward.""" inp = args.inp use_bias = args.use_bias is_first_microbatch = args.is_first_microbatch fp8 = args.fp8 input_quantizers = args.input_quantizers - weight_quantizers = args.weight_quantizers activation_dtype = args.activation_dtype is_grad_enabled = args.is_grad_enabled weight_workspaces = args.weight_workspaces @@ -801,13 +1307,16 @@ def _grouped_linear_fused_forward(args: GroupedLinearFwdArgs) -> Tuple[Any, ...] is_dist_weight = is_distributed_weight(weights[0]) if is_dist_weight: weights = materialize_weight_for_forward(weights) + _grouped_linear_configure_quantizers(args, weights) + weight_quantizers = args.weight_quantizers biases = args.biases out = args.out m_splits = args.m_splits_tensor num_gemms = len(m_splits) device = inp.device - in_features = weights[0].size(-1) - out_features = weights[0].size(-2) + weight_shape = weights[0].shape + in_features = weight_shape[-1] + out_features = weight_shape[-2] if inp.size(-1) != in_features: raise ValueError( f"Input tensor (shape={tuple(inp.size())}) is not compatible with " @@ -896,13 +1405,16 @@ def _grouped_linear_fused_forward(args: GroupedLinearFwdArgs) -> Tuple[Any, ...] dtype=activation_dtype, ) + fprop_use_split_accumulator = args.fprop_use_split_accumulator + if fp8 and hasattr(args.fp8_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = args.fp8_recipe.fp8_gemm_fprop.use_split_accumulator general_grouped_gemm_for_grouped_tensor( weights_for_gemm, grouped_x, grouped_out, layout="TN", bias=grouped_bias, - use_split_accumulator=args.fprop_use_split_accumulator, + use_split_accumulator=fprop_use_split_accumulator, ) tensors_to_save = None @@ -942,8 +1454,8 @@ def _grouped_linear_fused_forward(args: GroupedLinearFwdArgs) -> Tuple[Any, ...] return out.view(-1, *inp.shape[1:-1], out.shape[-1]), new_workspaces, tensors_to_save -def _grouped_linear_fused_setup( - bwd_args: GroupedLinearFusedBwdArgs, +def _grouped_linear_graph_safe_setup( + bwd_args: GroupedLinearGraphSafeBwdArgs, fwd_args: GroupedLinearFwdArgs, out_features: int, ) -> None: @@ -960,6 +1472,7 @@ def _grouped_linear_fused_setup( bwd_args.single_grouped_weight = fwd_args.single_grouped_weight bwd_args.single_grouped_bias = fwd_args.single_grouped_bias bwd_args.is_first_microbatch = fwd_args.is_first_microbatch + bwd_args.fp8_recipe = fwd_args.fp8_recipe bwd_args.dgrad_use_split_accumulator = fwd_args.dgrad_use_split_accumulator bwd_args.wgrad_use_split_accumulator = fwd_args.wgrad_use_split_accumulator bwd_args.fuse_wgrad_accumulation = fwd_args.fuse_wgrad_accumulation @@ -986,8 +1499,8 @@ def _grouped_linear_fused_setup( ] -def _grouped_linear_fused_backward( - args: GroupedLinearFusedBwdArgs, +def _grouped_linear_graph_safe_backward( + args: GroupedLinearGraphSafeBwdArgs, ) -> Tuple[Optional[torch.Tensor], List[Optional[torch.Tensor]], List[Optional[torch.Tensor]]]: """Compute gradients with the grouped-tensor kernels.""" grad_output = args.grad_output @@ -1104,13 +1617,19 @@ def _grouped_linear_fused_backward( if args.single_grouped_bias: grad_bias_args = [dbias_packed.to(dtype=args.activation_dtype)] else: - grad_bias_args = [dbias_packed[i].to(dtype=args.activation_dtype) for i in range(N)] + grad_bias_args = [ + dbias_packed[i].to(dtype=args.activation_dtype, copy=args.compiled_op) + for i in range(N) + ] else: num_bias_args = 1 if args.single_grouped_bias else N grad_bias_args = [None] * num_bias_args dgrad = None if args.requires_dgrad: + dgrad_use_split_accumulator = args.dgrad_use_split_accumulator + if args.fp8 and hasattr(args.fp8_recipe, "fp8_gemm_dgrad"): + dgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_dgrad.use_split_accumulator for weight in weight_tensors: if isinstance(weight, QuantizedTensorStorage): weight.update_usage(columnwise_usage=True) @@ -1134,7 +1653,7 @@ def _grouped_linear_fused_backward( grouped_dy, grouped_dgrad, layout="NN", - use_split_accumulator=args.dgrad_use_split_accumulator, + use_split_accumulator=dgrad_use_split_accumulator, ) if is_dist_weight: @@ -1155,6 +1674,9 @@ def _grouped_linear_fused_backward( raise RuntimeError( "distributed-weight GroupedLinear requires delay_wgrad_compute=False." ) + wgrad_use_split_accumulator = args.wgrad_use_split_accumulator + if args.fp8 and hasattr(args.fp8_recipe, "fp8_gemm_wgrad"): + wgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_wgrad.use_split_accumulator if args.fuse_wgrad_accumulation: if args.single_grouped_weight: main_grad = main_grads[0] @@ -1183,14 +1705,24 @@ def _grouped_linear_fused_backward( grouped_wgrad.rowwise_data.view(N, args.out_features, args.in_features) ] else: - wgrad_packed = torch.empty( - N, - args.out_features, - args.in_features, - dtype=args.activation_dtype, - device=grad_output.device, - ) - wgrad_output = [wgrad_packed[i] for i in range(N)] + if args.compiled_op: + wgrad_output = [ + torch.empty( + (args.out_features, args.in_features), + dtype=args.activation_dtype, + device=grad_output.device, + ) + for _ in range(N) + ] + else: + wgrad_packed = torch.empty( + N, + args.out_features, + args.in_features, + dtype=args.activation_dtype, + device=grad_output.device, + ) + wgrad_output = [wgrad_packed[i] for i in range(N)] wgrad_list = wgrad_output accumulate = ( @@ -1198,7 +1730,6 @@ def _grouped_linear_fused_backward( if not getattr(args, "origin_weights_overwrite_main_grad", False) else False ) - wgrad_use_split_accumulator = args.wgrad_use_split_accumulator def grouped_gemm_wgrad(inputmats, grad_output_mats, grad_weights): general_grouped_gemm_for_grouped_tensor( @@ -1231,6 +1762,192 @@ def grouped_gemm_wgrad(inputmats, grad_output_mats, grad_weights): return dgrad, wgrad_list, grad_bias_args +# --------------------------------------------------------------------------- # +# Fused GroupedTensor path under torch.compile +# --------------------------------------------------------------------------- # + + +def _grouped_linear_fused_forward_impl(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: + """Encode the shared grouped forward's saved tensors without aliases.""" + out, new_workspaces, saved = _grouped_linear_graph_safe_forward(args) + tensors_to_save = None + if saved is not None: + inputmat = saved[0] + if inputmat is not None: + if args.save_original_input or ( + not args.fp8 and args.inp.dtype == args.activation_dtype + ): + inputmat = SavedTensorRef("inp") + elif not args.fp8: + inputmat = inputmat.rowwise_data + else: + # Dynamic rows and split metadata are restored inside backward. + inputmat.logical_shape = (0, args.weights[0].shape[-1]) + inputmat.first_dims = None + inputmat.tensor_offsets = None + weights = _saved_grouped_weights(args, new_workspaces, saved[1 : 1 + len(args.weights)]) + tensors_to_save = (inputmat, *weights) + return (out, *new_workspaces, tensors_to_save, None) + + +def _fake_grouped_input(args): + """Describe the saved grouped activation using the storage's own layout.""" + if not args.weights_requires_grad: + return None + if args.save_original_input or (not args.fp8 and args.inp.dtype == args.activation_dtype): + return SavedTensorRef("inp") + tokens = math.prod(args.inp.shape[:-1]) + in_features = args.weights[0].shape[-1] + total = tokens * in_features + + def spec(numel, dtype): + return TensorSpec(shape=(numel,), dtype=dtype, device=args.inp.device) + + if not args.fp8: + return spec(total, args.activation_dtype) + quantizer = args.input_quantizers[0] + block_scaled = isinstance(quantizer, (MXFP8Quantizer, NVFP4Quantizer)) + buffers = {} + if block_scaled: + is_nvfp4 = isinstance(quantizer, NVFP4Quantizer) + buffers["columnwise_data"] = spec(total // 2 if is_nvfp4 else total, torch.uint8) + buffers["columnwise_scale_inv"] = spec( + math.prod(quantizer.get_scale_shape((tokens, in_features), True)), torch.uint8 + ) + if is_nvfp4: + buffers["amax"] = spec(args.num_gemms, torch.float32) + buffers["columnwise_amax"] = spec(args.num_gemms, torch.float32) + else: + # Hopper retains the columnwise operand; Blackwell reuses rowwise data. + if is_non_tn_fp8_gemm_supported(): + buffers["data"] = spec(total, torch.uint8) + buffers["scale_inv"] = spec(args.num_gemms, torch.float32) + else: + buffers["columnwise_data"] = spec(total, torch.uint8) + buffers["columnwise_scale_inv"] = spec(args.num_gemms, torch.float32) + buffers["amax"] = spec(args.num_gemms, torch.float32) + buffers["scale"] = spec(args.num_gemms, torch.float32) + return to_tensor_spec( + GroupedTensorStorage( + shape=(0, in_features), + dtype=args.activation_dtype, + num_tensors=args.num_gemms, + quantizer=quantizer, + with_gemm_swizzled_scales=block_scaled, + **buffers, + ) + ) + + +def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: + """Shape/metadata-only twin of :func:`_grouped_linear_fused_forward_impl`.""" + inp = args.inp + out_features = args.weights[0].shape[-2] + activation_dtype = args.activation_dtype + fp8 = args.fp8 + is_grad_enabled = args.is_grad_enabled + weight_requires_grad = args.weights_requires_grad + device = inp.device + + if fp8: + input_quantizer = args.input_quantizers[0] + input_quantizer.set_usage( + rowwise=True, + columnwise=is_grad_enabled and weight_requires_grad and not args.save_original_input, + ) + input_quantizer.optimize_for_gemm = True + + if fp8: + for quantizer in args.weight_quantizers: + quantizer.set_usage( + rowwise=True, columnwise=is_grad_enabled and args.input_requires_grad + ) + new_workspaces, weights_saved = _fake_grouped_weights(args, args.weight_quantizers) + + out = TensorSpec( + shape=(*tuple(inp.shape[:-1]), out_features), + dtype=activation_dtype, + requires_grad=is_grad_enabled + and (args.input_requires_grad or weight_requires_grad or args.bias_requires_grad), + device=device, + ) + + tensors_to_save = None + if is_grad_enabled: + if not args.input_requires_grad: + weights_saved = [None] * len(args.weights) + tensors_to_save = (_fake_grouped_input(args), *weights_saved) + return (out, *new_workspaces, tensors_to_save, None) + + +def _grouped_linear_fused_setup_ctx( + bwd_args: GroupedLinearGraphSafeBwdArgs, + fwd_args: GroupedLinearFwdArgs, + fwd_outputs: Tuple[Any, ...], + ctx_attrs: Dict, + tensors_to_save_from_forward: Tuple[Any, ...], +) -> Tuple[Any, ...]: + """Populate the fused backward args and pass through its saved tensors.""" + del ctx_attrs + _grouped_linear_graph_safe_setup(bwd_args, fwd_args, fwd_outputs[0].shape[-1]) + bwd_args.compiled_op = fwd_args.compiled_op + return tensors_to_save_from_forward + + +def _grouped_linear_fused_backward_impl( + args: GroupedLinearGraphSafeBwdArgs, +) -> Tuple[Optional[torch.Tensor], List[Optional[torch.Tensor]], List[Optional[torch.Tensor]]]: + """Rebuild the op's saved grouped input and run shared backward.""" + args.m_splits_tensor = args.m_splits_tensor.to(device=args.grad_output.device) + args.base_split_offsets = tex.splits_to_offsets(args.m_splits_tensor, 1) + args.input_tensor_offsets = args.base_split_offsets * args.in_features + args.output_tensor_offsets = args.base_split_offsets * args.out_features + if args.inputmat is not None and not args.save_original_input: + if args.fp8: + args.inputmat.logical_shape = (math.prod(args.grad_output.shape[:-1]), args.in_features) + args.inputmat.first_dims = args.m_splits_tensor + args.inputmat.tensor_offsets = args.input_tensor_offsets + else: + args.inputmat = _GroupedLinear._make_grouped_tensor( + args.inputmat.reshape(-1, args.in_features), + num_gemms=args.num_gemms, + split_sizes=args.m_splits_tensor, + tensor_offsets=args.input_tensor_offsets, + last_dim=args.in_features, + dtype=args.activation_dtype, + ) + return _grouped_linear_graph_safe_backward(args) + + +def _grouped_linear_fused_backward_fake( + args: GroupedLinearGraphSafeBwdArgs, +) -> Tuple[Optional[TensorSpec], List[Optional[TensorSpec]], List[Optional[TensorSpec]]]: + """Allocation-free fake of :func:`_grouped_linear_fused_backward_impl`.""" + + if args.fp8: + grad_output_quantizer = args.grad_output_quantizers[0] + grad_output_quantizer.set_usage( + rowwise=args.requires_dgrad, + columnwise=args.weights_requires_grad, + ) + + return _fake_grouped_grads(args, args.in_features, args.use_bias) + + +# Custom op for the fused GroupedTensor path under ``torch.compile``. +_grouped_linear_fused_op = register_custom_op_with_autograd( + op_name="grouped_linear_fused", + input_tensors_for_grad=["inp", "weights", "biases"], + fwd_arg_type=GroupedLinearFwdArgs, + fwd_impl=_grouped_linear_fused_forward_impl, + fwd_fake_impl=_grouped_linear_fused_forward_fake, + setup_context=_grouped_linear_fused_setup_ctx, + bwd_arg_type=GroupedLinearGraphSafeBwdArgs, + bwd_impl=_grouped_linear_fused_backward_impl, + bwd_fake_impl=_grouped_linear_fused_backward_fake, +) + + @no_torch_dynamo() def _grouped_linear_eager( inp: torch.Tensor, @@ -1367,8 +2084,15 @@ def _prepare_weights_for_grouped_tensor_gemm( workspace = weight_workspaces[0] if weight_workspaces else None if workspace is not None and ( - workspace.quantizer is not weight_quantizer - or (columnwise_usage and workspace.columnwise_data is None) + workspace.quantizer != weight_quantizer + or ( + columnwise_usage + and workspace.columnwise_data is None + and not ( + isinstance(weight_quantizer, Float8CurrentScalingQuantizer) + and is_non_tn_fp8_gemm_supported() + ) + ) ): workspace = None @@ -1376,6 +2100,7 @@ def _prepare_weights_for_grouped_tensor_gemm( raise RuntimeError("Single grouped weight has no rowwise storage to quantize.") source = weight.rowwise_data.view(weight.logical_shape) update_workspace = is_first_microbatch is None or is_first_microbatch + new_workspace = workspace is None if workspace is None: if cache_weight: # Match quantize_weight(): persistent workspaces must be Tensor subclasses @@ -1391,6 +2116,7 @@ def _prepare_weights_for_grouped_tensor_gemm( if cache_weight: weight_quantizer.internal = saved_internal elif skip_fp8_weight_update is not None or update_workspace: + workspace.quantizer = weight_quantizer grouped_weight = tex.group_quantize( source, weight_quantizer, @@ -1402,7 +2128,7 @@ def _prepare_weights_for_grouped_tensor_gemm( else: grouped_weight = workspace - if cache_weight: + if cache_weight and new_workspace: new_workspaces[0] = grouped_weight return grouped_weight, new_workspaces @@ -1502,10 +2228,10 @@ def _prepare_bias_for_grouped_tensor_gemm( @staticmethod def _forward_grouped_tensor(ctx, fwd_args: GroupedLinearFwdArgs) -> Tuple[torch.Tensor, list]: """Run the grouped forward and persist its autograd state.""" - out, new_workspaces, saved = _grouped_linear_fused_forward(fwd_args) + out, new_workspaces, saved = _grouped_linear_graph_safe_forward(fwd_args) if ctx is not None: - bwd_args = GroupedLinearFusedBwdArgs() - _grouped_linear_fused_setup(bwd_args, fwd_args, out.shape[-1]) + bwd_args = GroupedLinearGraphSafeBwdArgs() + _grouped_linear_graph_safe_setup(bwd_args, fwd_args, out.shape[-1]) tensors_to_save, tensor_objects = prepare_for_saving(*saved) ctx.save_for_backward(*tensors_to_save) ctx.tensor_objects = tensor_objects @@ -1542,12 +2268,22 @@ def forward( if fwd_args.use_grouped_tensor_path: return _GroupedLinear._forward_grouped_tensor(ctx, fwd_args) - out, new_workspaces, saved = _grouped_linear_forward_impl(fwd_args) + outputs = _grouped_linear_forward_impl(fwd_args) + out = outputs[0] + new_workspaces = list(outputs[1 : 1 + num_gemms]) + tensors_to_save_from_forward, ctx_attrs = outputs[-2], outputs[-1] + if ctx is not None: ctx.use_grouped_tensor_path = False bwd_args = GroupedLinearBwdArgs() - _grouped_linear_setup_ctx(bwd_args, fwd_args) - tensors_to_save, tensor_objects = prepare_for_saving(*saved) + tensors_to_save_from_setup = _grouped_linear_setup_ctx( + bwd_args, + fwd_args, + (out, *new_workspaces), + ctx_attrs, + tensors_to_save_from_forward, + ) + tensors_to_save, tensor_objects = prepare_for_saving(*tensors_to_save_from_setup) ctx.save_for_backward(*tensors_to_save) ctx.tensor_objects = tensor_objects ctx.backward_objects = bwd_args @@ -1570,7 +2306,7 @@ def backward( bwd_args.grad_output = grad_output bwd_args.setup_saved_tensors(ctx) if ctx.use_grouped_tensor_path: - dgrad, wgrad_list, grad_biases = _grouped_linear_fused_backward(bwd_args) + dgrad, wgrad_list, grad_biases = _grouped_linear_graph_safe_backward(bwd_args) if dgrad is not None: dgrad = dgrad.view(ctx.inp_shape) else: @@ -1589,6 +2325,12 @@ class GroupedLinear(TransformerEngineBaseModule): """Applies linear transformations to the incoming data list :math:`y_i = x_iA_i^T + b_i` in a grouped way. + ``torch.compile(fullgraph=True)`` supports discrete expert parameters with host + split lists. The native grouped-tensor path supports CUDA split tensors with + BF16/FP16, FP8 current scaling, MXFP8, or NVFP4, including ``save_original_input=True``. + Single grouped weights and biases are supported for the native path's compatible recipes. + Unsupported configurations fall back to eager when graph breaks are allowed. + Parameters ---------- num_gemms : int @@ -2039,6 +2781,16 @@ def reset_parameters(self, defer_init=False): self.make_grouped_weights(defer_init=defer_init) elif self.single_grouped_bias: self._make_grouped_biases() + if not defer_init: + # Allocate the process-global grouped cuBLAS workspace eagerly: + # under torch.compile the first grouped GEMM can run inside + # CUDA-graph capture, and a workspace first allocated there would + # live in the graph pool. + weight = getattr(self, "weight0", None) + if weight is None: + weight = getattr(self, "weight", None) + if weight is not None and weight.device.type == "cuda": + get_cublas_workspace(weight.device.index, False, True) def set_tensor_parallel_attributes(self, defer_init=False) -> None: """Set attributes needed for TP""" @@ -2203,7 +2955,6 @@ def _load_from_state_dict( error_msgs, ) - @no_torch_dynamo() def forward( self, inp: torch.Tensor, @@ -2259,7 +3010,9 @@ def forward( "use_grouped_tensor=True; the split-quantize path only supports discrete " "parameters." ) + m_splits_host: Optional[Tuple[int, ...]] = None if not isinstance(m_splits, torch.Tensor): + m_splits_host = tuple(int(s) for s in m_splits) # Convert list of ints to tensor for backward compatibility m_splits = torch.tensor(m_splits, dtype=torch.int64, device="cpu") elif m_splits.dtype != torch.int64: @@ -2319,6 +3072,10 @@ def forward( for q in weight_quantizers: q.optimize_for_gemm = optimize_for_gemm + use_compiled_op = torch.compiler.is_compiling() and _grouped_linear_op is not None + if _grouped_linear_op is None and torch.compiler.is_compiling(): + warn_if_compile_disabled() + cache_weight = is_first_microbatch is not None if self.single_grouped_weight: weight_workspaces = [self._fp8_workspaces.get("weight")] if cache_weight else [None] @@ -2330,18 +3087,9 @@ def forward( ) weight_requires_grad = weight_tensors[0].requires_grad - fprop_use_split_accumulator = _2X_ACC_FPROP - dgrad_use_split_accumulator = _2X_ACC_DGRAD - wgrad_use_split_accumulator = _2X_ACC_WGRAD if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override - if hasattr(_recipe, "fp8_gemm_fprop"): - fprop_use_split_accumulator = _recipe.fp8_gemm_fprop.use_split_accumulator - if hasattr(_recipe, "fp8_gemm_dgrad"): - dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator - if hasattr(_recipe, "fp8_gemm_wgrad"): - wgrad_use_split_accumulator = _recipe.fp8_gemm_wgrad.use_split_accumulator else: _recipe = None backward_override = None @@ -2424,8 +3172,8 @@ def forward( fwd_args = GroupedLinearFwdArgs( # tensors inp=inp, - weights=list(weight_tensors), - biases=list(bias_tensors), + weights=weight_tensors, + biases=bias_tensors, weight_workspaces=weight_workspaces, out=out, dgrad_out=dgrad_out, @@ -2434,6 +3182,7 @@ def forward( # requires_grad flags input_requires_grad=inp.requires_grad, weights_requires_grad=weight_requires_grad, + bias_requires_grad=(bias_tensors[0].requires_grad if self.apply_bias else False), # quantizers input_quantizers=input_quantizers, weight_quantizers=weight_quantizers, @@ -2442,7 +3191,7 @@ def forward( grad_weight_quantizers=grad_weight_quantizers, grad_output_quantizers=grad_output_quantizers, # split geometry - m_splits=None, + m_splits=m_splits_host, num_gemms=num_gemms, # numerical / dtype config activation_dtype=self.activation_dtype, @@ -2450,9 +3199,7 @@ def forward( fp8_calibration=self.fp8_calibration, save_original_input=save_original_input, backward_override=backward_override, - fprop_use_split_accumulator=fprop_use_split_accumulator, - dgrad_use_split_accumulator=dgrad_use_split_accumulator, - wgrad_use_split_accumulator=wgrad_use_split_accumulator, + fp8_recipe=_recipe, debug=debug, # weight-workspace caching is_first_microbatch=is_first_microbatch, @@ -2469,13 +3216,54 @@ def forward( is_grad_enabled=is_grad_enabled, ) - out, new_workspaces = _grouped_linear_eager( - inp, - m_splits, - fwd_args, - (*weight_tensors, *bias_tensors), - is_grad_enabled, - ) + if use_compiled_op: + compiled_op = ( + _grouped_linear_fused_op if use_grouped_tensor_path else _grouped_linear_op + ) + fallback_reason = ( + fwd_args.compile_unsupported_reason() + if compiled_op is not None + else "custom-op registration unavailable" + ) + if fallback_reason is not None: + torch._dynamo.graph_break( + msg=f"te.GroupedLinear falling back to eager: {fallback_reason}" + ) + warn_compile_eager_fallback(fallback_reason) + use_compiled_op = False + + if use_compiled_op: + fwd_args.compiled_op = True + if self.fp8: + if hasattr(_recipe, "fp8_gemm_fprop"): + fwd_args.fprop_use_split_accumulator = ( + _recipe.fp8_gemm_fprop.use_split_accumulator + ) + if hasattr(_recipe, "fp8_gemm_dgrad"): + fwd_args.dgrad_use_split_accumulator = ( + _recipe.fp8_gemm_dgrad.use_split_accumulator + ) + if hasattr(_recipe, "fp8_gemm_wgrad"): + fwd_args.wgrad_use_split_accumulator = ( + _recipe.fp8_gemm_wgrad.use_split_accumulator + ) + fwd_args.fp8_recipe = None + if use_grouped_tensor_path: + fwd_args.m_splits = [] + else: + fwd_args.m_splits_tensor = None + check_grouped_gemm_dims(inp, weight_tensors[0], fwd_args.m_splits, self.fp8) + outputs = compiled_op(fwd_args) + out = outputs[0] + new_workspaces = list(outputs[1:]) + else: + out, new_workspaces = _grouped_linear_eager( + inp, + m_splits, + fwd_args, + (*weight_tensors, *bias_tensors), + is_grad_enabled, + ) if cache_weight: for i, ws in enumerate(new_workspaces): diff --git a/transformer_engine/pytorch/module/layernorm_linear.py b/transformer_engine/pytorch/module/layernorm_linear.py index e93d8c9641b..61b2a155449 100644 --- a/transformer_engine/pytorch/module/layernorm_linear.py +++ b/transformer_engine/pytorch/module/layernorm_linear.py @@ -1437,6 +1437,7 @@ def wgrad_gemm( wgrad = get_dummy_wgrad( list(main_grad.shape), origin_weight.dtype, + zero_tensor=getattr(origin_weight, "zero_tensor_dummy_wgrad", False), ) elif args.fuse_wgrad_accumulation: wgrad = None diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index ad5864d4a60..47ca3a87165 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -21,6 +21,7 @@ from .base import ( fill_userbuffers_buffer_for_all_gather, _ub_communicators, + get_dummy_wgrad, get_ub, get_ub_is_fp8, is_ub_initialized, @@ -2003,20 +2004,12 @@ def fc1_wgrad_gemm( fc1_weight_python_object, "grad_added_to_main_grad" ): fc1_weight_python_object.grad_added_to_main_grad = True - if getattr(fc1_weight_python_object, "zero_out_wgrad", False): - fc1_wgrad = torch.zeros( - fc1_weight_main_grad.shape, - dtype=fc1_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - fc1_wgrad = torch.empty( - fc1_weight_main_grad.shape, - dtype=fc1_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) + fc1_wgrad = get_dummy_wgrad( + list(fc1_weight_main_grad.shape), + fc1_weight_python_object.dtype, + zero=getattr(fc1_weight_python_object, "zero_out_wgrad", False), + zero_tensor=getattr(fc1_weight_python_object, "zero_tensor_dummy_wgrad", False), + ) elif args.fuse_wgrad_accumulation: fc1_wgrad = None else: @@ -2028,20 +2021,12 @@ def fc1_wgrad_gemm( fc2_weight_python_object, "grad_added_to_main_grad" ): fc2_weight_python_object.grad_added_to_main_grad = True - if getattr(fc2_weight_python_object, "zero_out_wgrad", False): - fc2_wgrad = torch.zeros( - fc2_weight_main_grad.shape, - dtype=fc2_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - fc2_wgrad = torch.empty( - fc2_weight_main_grad.shape, - dtype=fc2_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) + fc2_wgrad = get_dummy_wgrad( + list(fc2_weight_main_grad.shape), + fc2_weight_python_object.dtype, + zero=getattr(fc2_weight_python_object, "zero_out_wgrad", False), + zero_tensor=getattr(fc2_weight_python_object, "zero_tensor_dummy_wgrad", False), + ) elif args.fuse_wgrad_accumulation: fc2_wgrad = None else: diff --git a/transformer_engine/pytorch/module/linear.py b/transformer_engine/pytorch/module/linear.py index 7f00e19644a..73c9503026f 100644 --- a/transformer_engine/pytorch/module/linear.py +++ b/transformer_engine/pytorch/module/linear.py @@ -1705,6 +1705,9 @@ def wgrad_gemm( wgrad = get_dummy_wgrad( wgrad_shape, origin_weight_python_object.dtype, + zero_tensor=getattr( + origin_weight_python_object, "zero_tensor_dummy_wgrad", False + ), ) elif bwd_args.fuse_wgrad_accumulation: wgrad = None diff --git a/transformer_engine/pytorch/ops/_common.py b/transformer_engine/pytorch/ops/_common.py index 19a5eaf473b..cf4e66023a6 100644 --- a/transformer_engine/pytorch/ops/_common.py +++ b/transformer_engine/pytorch/ops/_common.py @@ -11,6 +11,7 @@ import torch from transformer_engine_torch import FP8TensorMeta +from ..dynamo.tensor_spec import TensorSpec from ..ep import ( EpBuffer, EpConfig, @@ -156,8 +157,10 @@ def validate_or_alloc_output( return buffer -def is_quantized_tensor(tensor: torch.Tensor | QuantizedTensorStorage) -> bool: - """Check if tensor is a quantized tensor""" +def is_quantized_tensor(tensor: torch.Tensor | QuantizedTensorStorage | TensorSpec) -> bool: + """Check whether a tensor or tensor spec represents quantized data.""" + if isinstance(tensor, TensorSpec): + return tensor.is_quantized return isinstance(tensor, QuantizedTensorStorage) @@ -282,7 +285,8 @@ def get_dummy_wgrads_for_params( For each parameter that exposes ``grad_added_to_main_grad``, set the flag to ``True`` and return a dummy wgrad tensor (zeroed if - ``zero_out_wgrad`` is also set on the parameter). For parameters without + ``zero_out_wgrad`` is also set on the parameter, otherwise a ZeroTensor if + ``zero_tensor_dummy_wgrad`` is set). For parameters without the flag, the corresponding entry is ``None``. The returned list has the same length and order as ``weight_params``. @@ -298,6 +302,7 @@ def get_dummy_wgrads_for_params( list(wp.size()), wp.dtype, zero=getattr(wp, "zero_out_wgrad", False), + zero_tensor=getattr(wp, "zero_tensor_dummy_wgrad", False), ) ) else: diff --git a/transformer_engine/pytorch/ops/basic/add_extra_input.py b/transformer_engine/pytorch/ops/basic/add_extra_input.py index 9af399f2dc6..87a4afee5a1 100644 --- a/transformer_engine/pytorch/ops/basic/add_extra_input.py +++ b/transformer_engine/pytorch/ops/basic/add_extra_input.py @@ -44,7 +44,7 @@ def __init__(self, *, in_place: bool = False): def op_forward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_forward` instead of `op_forward`." @@ -52,7 +52,7 @@ def op_forward(self, *args, **kwargs) -> None: def op_backward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_backward` instead of `op_backward`." diff --git a/transformer_engine/pytorch/ops/basic/basic_linear.py b/transformer_engine/pytorch/ops/basic/basic_linear.py index cb429055a4f..a7b09198250 100644 --- a/transformer_engine/pytorch/ops/basic/basic_linear.py +++ b/transformer_engine/pytorch/ops/basic/basic_linear.py @@ -5,14 +5,16 @@ """Fusible operation for linear layer without bias.""" from __future__ import annotations -from collections.abc import Callable, Iterable +from collections.abc import Callable import contextlib +from dataclasses import dataclass import math from typing import Any, Optional import torch -from ...cpp_extensions import general_gemm +from ...cpp_extensions import general_gemm, get_cublas_workspace +from ...dynamo import TensorOrQuantized, TensorSpec from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload from ...distributed import ( CudaRNGStatesTracker, @@ -35,7 +37,7 @@ clear_tensor_data, devices_match, ) -from ..op import BasicOperation, OperationContext +from ..op import BasicOperation from .._common import ( get_accumulate_flag_in_param, get_dummy_wgrads_for_params, @@ -45,6 +47,75 @@ ) +@dataclass(slots=True) +class BasicLinearFwdArgs: + """Tensor inputs and configuration for the linear forward custom op.""" + + input_: TensorOrQuantized + weight: TensorOrQuantized + dtype: torch.dtype + input_requires_grad: bool + weight_requires_grad: bool + with_quantized_compute: bool + backward_override: Optional[str] + input_quantizer: Optional[Quantizer] + weight_quantizer: Optional[Quantizer] + output_quantizer: Optional[Quantizer] + grad_output_quantizer: Optional[Quantizer] + grad_input_quantizer: Optional[Quantizer] + tensor_parallel_mode: Optional[str] + tensor_parallel_group: Optional[torch.distributed.ProcessGroup] + tensor_parallel_size: int + sequence_parallel: bool + bias: Optional[torch.Tensor] = None + + +@dataclass(slots=True) +class BasicLinearBwdArgs: + """Saved tensors and configuration for the linear backward custom op.""" + + grad_output: TensorOrQuantized + input_: Optional[TensorOrQuantized] + weight: Optional[TensorOrQuantized] + dtype: torch.dtype + input_requires_grad: bool + weight_requires_grad: bool + with_quantized_compute: bool + input_quantizer: Optional[Quantizer] + weight_quantizer: Optional[Quantizer] + grad_output_quantizer: Optional[Quantizer] + grad_input_quantizer: Optional[Quantizer] + tensor_parallel_mode: Optional[str] + tensor_parallel_group: Optional[torch.distributed.ProcessGroup] + tensor_parallel_size: int + sequence_parallel: bool + grad_weight: Optional[torch.Tensor] + accumulate_into_grad_weight: bool + # Original parameter for eager-only main_grad flags and dummy wgrad. + weight_param: Optional[torch.Tensor] = None + + +def _save_quantized_input(args: BasicLinearFwdArgs) -> bool: + return ( + args.with_quantized_compute + and args.backward_override != "high_precision" + and args.weight_requires_grad + # Column SP gathers/quantizes a separate tensor; the saved local input + # is still the caller's tensor and must not be returned as fresh FP8 aux. + and not (args.tensor_parallel_mode == "column" and args.sequence_parallel) + and not is_quantized_tensor(args.input_) + ) + + +def _save_quantized_weight(args: BasicLinearFwdArgs) -> bool: + return ( + args.with_quantized_compute + and args.backward_override != "high_precision" + and args.input_requires_grad + and not is_quantized_tensor(args.weight) + ) + + def _wait_async(handle: Optional[Any]) -> None: """Wait for asynchronous communication to finish, if needed""" if handle is not None: @@ -340,6 +411,23 @@ def reset_parameters(self) -> None: if not isinstance(weight, torch.nn.Parameter): weight = torch.nn.Parameter(weight) self.weight = weight + self._prealloc_cublas_workspace() + + def _apply(self, *args, **kwargs): + out = super()._apply(*args, **kwargs) + self._prealloc_cublas_workspace() + return out + + def _prealloc_cublas_workspace(self) -> None: + """Keep the cached GEMM workspace outside CUDA graph private pools.""" + # pylint: disable=import-outside-toplevel + from torch._guards import detect_fake_mode + from torch._subclasses.fake_tensor import is_fake + + weight = self.weight + if weight.device.type != "cuda" or is_fake(weight) or detect_fake_mode(): + return + get_cublas_workspace(weight.device.index, False, False) def pre_first_fuser_forward(self) -> None: super().pre_first_fuser_forward() @@ -990,127 +1078,228 @@ def _functional_backward( _wait_async(dx_async) return dx, dw - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: + fwd_args_type = BasicLinearFwdArgs + bwd_args_type = BasicLinearBwdArgs - # Check which grads are required - input_requires_grad = ctx.requires_grad - weight_requires_grad = ctx.requires_grad and self.weight.requires_grad + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + if self._accumulate_into_main_grad: + return "accumulate_into_main_grad" + if is_cpu_offload_enabled(): + return "CPU offloading" + return super().compile_unsupported_reason(mode) - # Quantizers - input_quantizer = self.get_quantizer("forward", 0) - weight_quantizer = self.get_quantizer("forward", 1) - output_quantizer = next_op_input_quantizer - grad_output_quantizer = self.get_quantizer("backward", 0) - grad_input_quantizer = prev_op_grad_output_quantizer + def pack_forward_args( + self, + basic_op_ctxs, + input_, + *, + prev_op_grad_output_quantizer, + next_op_input_quantizer, + basic_op_kwargs, + **unused, # pylint: disable=unused-argument + ) -> BasicLinearFwdArgs: + if basic_op_kwargs[0]: + raise ValueError("BasicLinear forward does not expect keyword arguments") with_quantized_compute = FP8GlobalStateManager.is_fp8_enabled() - if with_quantized_compute: - backward_override = FP8GlobalStateManager.get_fp8_recipe().backward_override - else: - backward_override = None - - # Get autocast dtype if needed - if torch.is_autocast_enabled(): - dtype = torch.get_autocast_dtype("cuda") - else: - dtype = self.weight.dtype - - # Linear forward - output, x_local, w = BasicLinear._functional_forward( - input=input_, + backward_override = ( + FP8GlobalStateManager.get_fp8_recipe().backward_override + if with_quantized_compute + else None + ) + return BasicLinearFwdArgs( + input_=input_, weight=self.weight, - dtype=dtype, + dtype=( + torch.get_autocast_dtype("cuda") + if torch.is_autocast_enabled() + else self.weight.dtype + ), + input_requires_grad=basic_op_ctxs[0].requires_grad, + weight_requires_grad=basic_op_ctxs[0].requires_grad and self.weight.requires_grad, + with_quantized_compute=with_quantized_compute, + backward_override=backward_override, + input_quantizer=self.get_quantizer("forward", 0), + weight_quantizer=self.get_quantizer("forward", 1), + output_quantizer=next_op_input_quantizer, + grad_output_quantizer=self.get_quantizer("backward", 0), + grad_input_quantizer=prev_op_grad_output_quantizer, tensor_parallel_mode=self.tensor_parallel_mode, tensor_parallel_group=self.tensor_parallel_group, + tensor_parallel_size=self.tensor_parallel_size, sequence_parallel=self.sequence_parallel, - with_quantized_compute=with_quantized_compute, - backward_override=backward_override, - input_quantizer=input_quantizer, - weight_quantizer=weight_quantizer, - output_quantizer=output_quantizer, - input_requires_grad=input_requires_grad, - weight_requires_grad=weight_requires_grad, ) - # Save state for backward pass - if ctx.requires_grad: - if backward_override == "high_precision": - saved_input = input_ if weight_requires_grad else None - saved_weight = self.weight if input_requires_grad else None - else: - saved_input = x_local - saved_weight = w - if is_cpu_offload_enabled(): - # No special CPU offloading logic is needed for weights. saved_weight is - # either self.weight (nn.Parameter, auto-excluded from offload) or a - # workspace freshly created each forward pass. - mark_activation_offload(saved_input) - ctx.save_for_backward(saved_input, saved_weight) - ctx.with_quantized_compute = with_quantized_compute and backward_override is None - ctx.backward_override = backward_override - ctx.input_quantizer = input_quantizer - ctx.weight_quantizer = weight_quantizer - ctx.grad_output_quantizer = grad_output_quantizer - ctx.grad_input_quantizer = grad_input_quantizer - ctx.dtype = dtype - ctx.input_requires_grad = input_requires_grad - ctx.weight_requires_grad = weight_requires_grad - - return output - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, Iterable[Optional[torch.Tensor]]]: - - # Saved tensors from forward pass - (x_local, w) = ctx.saved_tensors + @classmethod + def forward_compute( + cls, args: BasicLinearFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + """Shared GEMM and saved tensors, optionally with a fused bias epilogue.""" + output, saved_input, saved_weight = cls._functional_forward( + input=args.input_, + weight=args.weight, + bias=args.bias, + dtype=args.dtype, + input_requires_grad=args.input_requires_grad, + weight_requires_grad=args.weight_requires_grad, + with_quantized_compute=args.with_quantized_compute, + backward_override=args.backward_override, + input_quantizer=args.input_quantizer, + weight_quantizer=args.weight_quantizer, + output_quantizer=args.output_quantizer, + tensor_parallel_mode=args.tensor_parallel_mode, + tensor_parallel_group=args.tensor_parallel_group, + sequence_parallel=args.sequence_parallel, + ) + return ( + output, + [()], + ( + saved_input if _save_quantized_input(args) else None, + saved_weight if _save_quantized_weight(args) else None, + ), + ) - # Megatron-LM wgrad fusion - # Note: Get grad tensor from param so we can accumulate - # directly into it. - accumulate_into_main_grad = self._accumulate_into_main_grad + @classmethod + def forward_compute_fake(cls, args: BasicLinearFwdArgs): + shape = list(args.input_.shape) + shape[-1] = args.weight.shape[0] + if args.sequence_parallel: + if args.tensor_parallel_mode == "column": + shape[0] *= args.tensor_parallel_size + elif args.tensor_parallel_mode == "row": + shape[0] //= args.tensor_parallel_size + output = TensorSpec( + shape=tuple(shape), + dtype=args.dtype, + device=args.input_.device, + quantizer=( + args.output_quantizer + if args.with_quantized_compute and args.tensor_parallel_mode != "row" + else None + ), + ) + saved = [] + for needed, value, quantizer in ( + (_save_quantized_input(args), args.input_, args.input_quantizer), + (_save_quantized_weight(args), args.weight, args.weight_quantizer), + ): + spec = None + if needed: + quantizer = quantizer.copy() + quantizer.set_usage( + rowwise=args.backward_override is not None, + columnwise=args.backward_override is None, + ) + spec = TensorSpec( + shape=value.shape, dtype=value.dtype, device=value.device, quantizer=quantizer + ) + saved.append(spec) + return output, [()], tuple(saved) + + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + ctx = basic_op_ctxs[0] + if not ctx.requires_grad: + return + saved_input, saved_weight = aux + if saved_input is None and args.weight_requires_grad: + saved_input = args.input_ + if saved_weight is None and args.input_requires_grad: + saved_weight = args.weight + if is_cpu_offload_enabled(): + mark_activation_offload(saved_input) + ctx.save_for_backward(saved_input, saved_weight) + ctx.with_quantized_compute = args.with_quantized_compute and args.backward_override is None + ctx.backward_override = args.backward_override + ctx.input_quantizer = args.input_quantizer + ctx.weight_quantizer = args.weight_quantizer + ctx.grad_output_quantizer = args.grad_output_quantizer + ctx.grad_input_quantizer = args.grad_input_quantizer + ctx.dtype = args.dtype + ctx.input_requires_grad = args.input_requires_grad + ctx.weight_requires_grad = args.weight_requires_grad + + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> BasicLinearBwdArgs: + ctx = basic_op_ctxs[0] grad_weight = None - if ctx.weight_requires_grad and accumulate_into_main_grad: - weight_param = self.weight - main_grad = get_main_grad_from_param(weight_param, op_label="BasicLinear") - accumulate_into_main_grad = get_accumulate_flag_in_param(weight_param) - grad_weight = main_grad.detach() - else: - accumulate_into_main_grad = False - - # Linear backward pass - grad_input, grad_weight = BasicLinear._functional_backward( + accumulate_into_grad_weight = False + if ctx.weight_requires_grad and self._accumulate_into_main_grad: + grad_weight = get_main_grad_from_param(self.weight, op_label="BasicLinear").detach() + accumulate_into_grad_weight = get_accumulate_flag_in_param(self.weight) + return BasicLinearBwdArgs( grad_output=grad_output, - input=x_local, - weight=w, + input_=ctx.saved_tensors[0], + weight=ctx.saved_tensors[1], + dtype=ctx.dtype, input_requires_grad=ctx.input_requires_grad, weight_requires_grad=ctx.weight_requires_grad, - dtype=ctx.dtype, - grad_weight=grad_weight, - accumulate_into_grad_weight=accumulate_into_main_grad, - tensor_parallel_mode=self.tensor_parallel_mode, - tensor_parallel_group=self.tensor_parallel_group, - sequence_parallel=self.sequence_parallel, with_quantized_compute=ctx.with_quantized_compute, input_quantizer=ctx.input_quantizer, weight_quantizer=ctx.weight_quantizer, grad_output_quantizer=ctx.grad_output_quantizer, grad_input_quantizer=ctx.grad_input_quantizer, + tensor_parallel_mode=self.tensor_parallel_mode, + tensor_parallel_group=self.tensor_parallel_group, + tensor_parallel_size=self.tensor_parallel_size, + sequence_parallel=self.sequence_parallel, + grad_weight=grad_weight, + accumulate_into_grad_weight=accumulate_into_grad_weight, + weight_param=self.weight if accumulate_into_grad_weight else None, ) - # Clear input tensor if possible - clear_tensor_data(x_local) - - # Megatron-LM wgrad fusion - # Note: Return dummy tensor for grad weight if needed. - if accumulate_into_main_grad: - grad_weight = get_dummy_wgrads_for_params([self.weight])[0] + @classmethod + def backward_compute(cls, args: BasicLinearBwdArgs, *, in_custom_op: bool = False): + dx, dw = cls._functional_backward( + grad_output=args.grad_output, + input=args.input_, + weight=args.weight, + dtype=args.dtype, + input_requires_grad=args.input_requires_grad, + weight_requires_grad=args.weight_requires_grad, + with_quantized_compute=args.with_quantized_compute, + input_quantizer=args.input_quantizer, + weight_quantizer=args.weight_quantizer, + grad_output_quantizer=args.grad_output_quantizer, + grad_input_quantizer=args.grad_input_quantizer, + tensor_parallel_mode=args.tensor_parallel_mode, + tensor_parallel_group=args.tensor_parallel_group, + sequence_parallel=args.sequence_parallel, + grad_weight=args.grad_weight, + accumulate_into_grad_weight=args.accumulate_into_grad_weight, + ) + if not in_custom_op: + clear_tensor_data(args.input_) + if args.accumulate_into_grad_weight: + dw = get_dummy_wgrads_for_params([args.weight_param])[0] + return dx, [(dw,)], [()] - return grad_input, [grad_weight] + @classmethod + def backward_compute_fake(cls, args: BasicLinearBwdArgs): + dx = None + if args.input_requires_grad: + shape = list(args.grad_output.shape) + shape[-1] = args.weight.shape[-1] + if args.sequence_parallel: + if args.tensor_parallel_mode == "column": + shape[0] //= args.tensor_parallel_size + elif args.tensor_parallel_mode == "row": + shape[0] *= args.tensor_parallel_size + dx = TensorSpec( + shape=tuple(shape), + dtype=args.dtype, + device=args.grad_output.device, + quantizer=( + args.grad_input_quantizer + if args.with_quantized_compute and args.tensor_parallel_mode != "column" + else None + ), + ) + dw = None + if args.weight_requires_grad: + dw = TensorSpec( + shape=(args.grad_output.shape[-1], args.input_.shape[-1]), + dtype=args.dtype, + device=args.grad_output.device, + ) + return dx, [(dw,)], [()] diff --git a/transformer_engine/pytorch/ops/basic/bias.py b/transformer_engine/pytorch/ops/basic/bias.py index 88f563b2c5b..70b34da1851 100644 --- a/transformer_engine/pytorch/ops/basic/bias.py +++ b/transformer_engine/pytorch/ops/basic/bias.py @@ -5,17 +5,36 @@ """Fusible operation for bias.""" from __future__ import annotations +from dataclasses import dataclass from typing import Optional import torch import transformer_engine_torch as tex +from ...dynamo import TensorOrQuantized, TensorSpec from ...quantization import FP8GlobalStateManager -from ..op import BasicOperation, OperationContext +from ..op import BasicOperation from ...utils import canonicalize_device, canonicalize_dtype from ...tensor import Quantizer +@dataclass(slots=True) +class BiasFwdArgs: + """Tensor inputs and backward quantizer for the bias forward custom op.""" + + input_: TensorOrQuantized + bias: torch.Tensor + grad_input_quantizer: Optional[Quantizer] + + +@dataclass(slots=True) +class BiasBwdArgs: + """Output gradient and optional quantizer for the bias backward custom op.""" + + grad_output: TensorOrQuantized + grad_input_quantizer: Optional[Quantizer] + + class Bias(BasicOperation): """Apply additive bias @@ -113,37 +132,75 @@ def pre_first_fuser_forward(self) -> None: if self.bias.device.type == "meta": self.reset_parameters() - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: - x = input_ - b = self.bias.view([1] * (x.dim() - 1) + [self.local_size]) - - if ctx.requires_grad: - ctx.grad_input_quantizer = prev_op_grad_output_quantizer - if FP8GlobalStateManager.is_fp8_enabled(): - fp8_recipe = FP8GlobalStateManager.get_fp8_recipe() - if fp8_recipe.backward_override is not None: - ctx.grad_input_quantizer = None - - return x + b - - def op_backward( + fwd_args_type = BiasFwdArgs + bwd_args_type = BiasBwdArgs + + def pack_forward_args( self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - dy = grad_output - if dy.dim() > 1: - quantizer = ctx.grad_input_quantizer - if quantizer is None: - db = dy.sum(tuple(range(dy.dim() - 1))) - else: - db, dy = tex.bgrad_quantize(dy, quantizer) - else: - db = dy - return dy, (db,) + basic_op_ctxs, + input_, + *, + prev_op_grad_output_quantizer, + basic_op_kwargs, + **unused, # pylint: disable=unused-argument + ) -> BiasFwdArgs: + if basic_op_kwargs[0]: + raise ValueError("Bias forward does not expect keyword arguments") + quantizer = prev_op_grad_output_quantizer + if FP8GlobalStateManager.is_fp8_enabled(): + if FP8GlobalStateManager.get_fp8_recipe().backward_override is not None: + quantizer = None + return BiasFwdArgs(input_, self.bias, quantizer) + + @classmethod + def forward_compute( + cls, args: BiasFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + return args.input_ + args.bias, [()], () + + @classmethod + def forward_compute_fake(cls, args: BiasFwdArgs): + return ( + TensorSpec( + shape=args.input_.shape, + dtype=torch.promote_types(args.input_.dtype, args.bias.dtype), + device=args.input_.device, + ), + [()], + (), + ) + + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + basic_op_ctxs[0].grad_input_quantizer = args.grad_input_quantizer + + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> BiasBwdArgs: + quantizer = basic_op_ctxs[0].grad_input_quantizer if len(grad_output.shape) > 1 else None + return BiasBwdArgs(grad_output, quantizer) + + @classmethod + def backward_compute( + cls, args: BiasBwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + dy = args.grad_output + if dy.dim() == 1: + return None, [(dy.clone(),)], [()] + if args.grad_input_quantizer is None: + return None, [(dy.sum(tuple(range(dy.dim() - 1))),)], [()] + db, dx = tex.bgrad_quantize(dy, args.grad_input_quantizer) + return dx, [(db,)], [()] + + @classmethod + def backward_compute_fake(cls, args: BiasBwdArgs): + dy = args.grad_output + dx = None + if len(dy.shape) > 1 and args.grad_input_quantizer is not None: + dx = TensorSpec( + shape=dy.shape, + dtype=dy.dtype, + device=dy.device, + quantizer=args.grad_input_quantizer, + ) + db = TensorSpec(shape=(dy.shape[-1],), dtype=dy.dtype, device=dy.device) + return dx, [(db,)], [()] diff --git a/transformer_engine/pytorch/ops/basic/make_extra_output.py b/transformer_engine/pytorch/ops/basic/make_extra_output.py index 47b7c6495de..dfe5c4b27d9 100644 --- a/transformer_engine/pytorch/ops/basic/make_extra_output.py +++ b/transformer_engine/pytorch/ops/basic/make_extra_output.py @@ -49,7 +49,7 @@ def __init__(self, *, in_place: bool = False): def op_forward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_forward` instead of `op_forward`." @@ -57,7 +57,7 @@ def op_forward(self, *args, **kwargs) -> None: def op_backward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_backward` instead of `op_backward`." diff --git a/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py b/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py index 6fa63675b4c..e45164a04c9 100644 --- a/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py +++ b/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py @@ -5,16 +5,13 @@ """Fused operation for forward GEMM + bias + activation.""" from __future__ import annotations -from collections.abc import Sequence -from typing import Any, Optional +from typing import Optional import torch -from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload -from ...quantization import FP8GlobalStateManager -from ...tensor import Quantizer from ..basic import BasicLinear, Bias -from ..op import FusedOperation, FusibleOperation, OperationContext +from ..basic.basic_linear import BasicLinearFwdArgs +from ..op import FusedOperation, FusibleOperation class ForwardLinearBiasActivation(FusedOperation): @@ -50,105 +47,47 @@ def __init__( # Index of each basic operations self._op_idxs: dict[str, Optional[int]] = op_idxs - def fuser_forward( - self, - basic_op_ctxs: list[OperationContext], - input_: torch.Tensor, - *, - basic_op_extra_inputs: list[tuple[torch.Tensor, ...]], - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - basic_op_kwargs: list[dict[str, Any]], - ) -> tuple[torch.Tensor, Sequence[Sequence[torch.Tensor]]]: - - # Get basic operations - idx = self._op_idxs["linear"] - linear_op = self.basic_ops[idx] - linear_op_ctx = basic_op_ctxs[idx] - if self._op_idxs["bias"] is None: - bias_op = None - bias_op_ctx = None - bias = None - else: - idx = self._op_idxs["bias"] - bias_op = self.basic_ops[idx] - bias_op_ctx = basic_op_ctxs[idx] - bias = bias_op.bias - if basic_op_kwargs[idx]: - raise ValueError("Bias operation forward does not expect keyword arguments") - if self._op_idxs["activation"] is None: - activation_op = None # pylint: disable=unused-variable - else: - raise NotImplementedError("Activations are not yet supported") + fwd_args_type = BasicLinearFwdArgs - # Check which grads are required - input_requires_grad = linear_op_ctx.requires_grad - weight_requires_grad = linear_op_ctx.requires_grad and linear_op.weight.requires_grad - - # Quantizers - input_quantizer = linear_op.get_quantizer("forward", 0) - weight_quantizer = linear_op.get_quantizer("forward", 1) - output_quantizer = next_op_input_quantizer - grad_output_quantizer = linear_op.get_quantizer("backward", 0) - grad_input_quantizer = prev_op_grad_output_quantizer - with_quantized_compute = FP8GlobalStateManager.is_fp8_enabled() - if with_quantized_compute: - backward_override = FP8GlobalStateManager.get_fp8_recipe().backward_override - else: - backward_override = None - - # Get autocast dtype if needed - if torch.is_autocast_enabled(): - dtype = torch.get_autocast_dtype("cuda") - else: - dtype = linear_op.weight.dtype - - # Linear forward - output, x_local, w = BasicLinear._functional_forward( - input=input_, - weight=linear_op.weight, - bias=bias, - dtype=dtype, - tensor_parallel_mode=linear_op.tensor_parallel_mode, - tensor_parallel_group=linear_op.tensor_parallel_group, - sequence_parallel=linear_op.sequence_parallel, - with_quantized_compute=with_quantized_compute, - backward_override=backward_override, - input_quantizer=input_quantizer, - weight_quantizer=weight_quantizer, - output_quantizer=output_quantizer, - input_requires_grad=input_requires_grad, - weight_requires_grad=weight_requires_grad, - ) + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + reason = super().compile_unsupported_reason(mode) + if reason is not None: + return reason + return self.basic_ops[0].compile_unsupported_reason(mode) - # Save state for backward pass - if linear_op_ctx.requires_grad: - if backward_override == "high_precision": - saved_input = input_ if weight_requires_grad else None - saved_weight = linear_op.weight if input_requires_grad else None - else: - saved_input = x_local - saved_weight = w - if is_cpu_offload_enabled(): - mark_activation_offload(saved_input) - linear_op_ctx.save_for_backward(saved_input, saved_weight) - linear_op_ctx.with_quantized_compute = ( - with_quantized_compute and backward_override is None + def pack_forward_args(self, basic_op_ctxs, input_, *, basic_op_kwargs, **kwargs): + if self._op_idxs["activation"] is not None: + raise NotImplementedError("Activations are not yet supported") + args = self.basic_ops[0].pack_forward_args( + basic_op_ctxs[:1], input_, basic_op_kwargs=basic_op_kwargs[:1], **kwargs + ) + if self._op_idxs["bias"] is not None: + idx = self._op_idxs["bias"] + if basic_op_kwargs[idx]: + raise ValueError("Bias forward does not expect keyword arguments") + args.bias = self.basic_ops[idx].bias + return args + + @classmethod + def forward_compute(cls, args: BasicLinearFwdArgs, *, in_custom_op: bool = False): + output, extras, aux = BasicLinear.forward_compute(args, in_custom_op=in_custom_op) + if args.bias is not None: + extras.append(()) + return output, extras, aux + + @classmethod + def forward_compute_fake(cls, args: BasicLinearFwdArgs): + output, extras, aux = BasicLinear.forward_compute_fake(args) + if args.bias is not None: + extras.append(()) + return output, extras, aux + + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + self.basic_ops[0].forward_setup_context(basic_op_ctxs[:1], args, aux) + if self._op_idxs["bias"] is not None: + basic_op_ctxs[self._op_idxs["bias"]].grad_input_quantizer = ( + args.grad_output_quantizer if args.backward_override is None else None ) - linear_op_ctx.backward_override = backward_override - linear_op_ctx.input_quantizer = input_quantizer - linear_op_ctx.weight_quantizer = weight_quantizer - linear_op_ctx.grad_output_quantizer = grad_output_quantizer - linear_op_ctx.grad_input_quantizer = grad_input_quantizer - linear_op_ctx.dtype = dtype - linear_op_ctx.input_requires_grad = input_requires_grad - linear_op_ctx.weight_requires_grad = weight_requires_grad - if bias_op is not None and bias_op_ctx.requires_grad: - bias_op_ctx.grad_input_quantizer = linear_op.get_grad_output_quantizer() - if backward_override is not None: - bias_op_ctx.grad_input_quantizer = None - - return output, [() for _ in range(len(self.basic_ops))] @staticmethod def fuse_forward_ops( diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index e30b98eea3f..eb336cbe6bc 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -249,6 +249,12 @@ def _deterministic_algorithms_required() -> bool: ) +_DETERMINISM_REQUESTED = ( + "Deterministic execution was requested (NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 or" + " torch.use_deterministic_algorithms)" +) + + def _wrap_single_quantized_as_grouped( tensor: torch.Tensor, quantized: MXFP8Tensor | NVFP4Tensor | NVFP4TensorStorage, @@ -1125,9 +1131,17 @@ def grouped_gemm_dactivation_kernel(cls) -> Callable: raise NotImplementedError @classmethod + @functools.lru_cache(maxsize=None) def grouped_gemm_dactivation_is_deterministic(cls) -> bool: - """Whether this op's dactivation kernel can produce a bit-exact ``dprob``.""" - return False + """Whether this op's dactivation wrapper accepts ``deterministic`` (feature-detected).""" + try: + kernel = cls.grouped_gemm_dactivation_kernel() + except ImportError: + return False + try: + return "deterministic" in inspect.signature(kernel).parameters + except (TypeError, ValueError): + return False @classmethod @functools.lru_cache(maxsize=None) @@ -1781,7 +1795,7 @@ def fuser_forward( fc1_w_scales = fc1_w_scales.view( 1, ceil_div(fc1_weight_shape[0], 128), - ceil_div(fc1_weight_shape[1], k_sf_divisor), + ceil_div(fc1_weight_k, k_sf_divisor), 32, 4, 4, @@ -1919,10 +1933,12 @@ def fuser_forward( fc2_bias_scale = fc2_bias_scale.to(dtype=torch.float32) fc2_out_buf = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) - if ( - use_dense_single_group - and grouped_fc2_x.columnwise_data is not None - and grouped_fc2_x.columnwise_scale_inv is not None + if use_dense_single_group and ( + not weight_requires_grad + or ( + grouped_fc2_x.columnwise_data is not None + and grouped_fc2_x.columnwise_scale_inv is not None + ) ): fc2_out_buf = _single_group_fc2_gemm( grouped_fc2_x, @@ -2454,12 +2470,10 @@ def fuser_backward( ) if not dprob_is_deterministic: raise RuntimeError( - "Deterministic execution was requested" - " (NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 or" - " torch.use_deterministic_algorithms), but the scale gradient (dprob) is" - " accumulated with nondeterministic atomics on this configuration." - " A bit-exact dprob requires the scaled-SReLU activation," - " nvidia-cudnn-frontend 1.28.0 or later, and an FC2 without scale_bias." + f"{_DETERMINISM_REQUESTED}, but the scale gradient (dprob) is accumulated" + " with nondeterministic atomics on this configuration. A bit-exact dprob" + " requires a cuDNN front-end whose dactivation wrapper accepts" + " `deterministic`, and an FC2 without scale_bias." ) scales_f32 = None scales_tensor = None @@ -2601,29 +2615,36 @@ def fuser_backward( fc2_weight_single._columnwise_scale_inv = original_columnwise_scale fc2_weight_single._with_gemm_swizzled_scales = original_swizzled - fc2_w_data = fc2_weight_single._columnwise_data.view(dtype=data_dtype) - fc2_w_data = fc2_w_data.view( - 1, - fc2_weight_shape[0], - fc2_weight_k, - ) - fc2_w_data = ( - fc2_w_data.permute(1, 2, 0) if use_nvfp4 else fc2_w_data.permute(2, 1, 0) - ) - fc2_w_scales = swizzled_columnwise_scale.view(dtype=scale_view_dtype) - fc2_w_scales = fc2_w_scales.view( - 1, - ceil_div(fc2_weight_shape[1], k_sf_divisor), - ceil_div(fc2_weight_shape[0], 128), - 32, - 4, - 4, - ) - fc2_w_scales = ( - fc2_w_scales.permute(3, 4, 2, 5, 1, 0) - if use_nvfp4 - else fc2_w_scales.permute(3, 4, 1, 5, 2, 0) - ) + if use_nvfp4: + # NVFP4 column-wise data is physically transposed, so it is already + # (in_features, out_features) and stays K-major. + fc2_w_data, fc2_w_scales = _convert_to_cudnn_grouped_gemm_tensor_format( + fc2_weight_single._columnwise_data, + swizzled_columnwise_scale, + data_dtype=data_dtype, + scale_dtype=scale_view_dtype, + valid_M_or_N=fc2_weight_shape[1], + k_logical=fc2_weight_shape[0], + sf_swizzled=True, + ) + else: + fc2_w_data = fc2_weight_single._columnwise_data.view(dtype=data_dtype) + fc2_w_data = fc2_w_data.view( + 1, + fc2_weight_shape[0], + fc2_weight_k, + ) + fc2_w_data = fc2_w_data.permute(2, 1, 0) + fc2_w_scales = swizzled_columnwise_scale.view(dtype=scale_view_dtype) + fc2_w_scales = fc2_w_scales.view( + 1, + ceil_div(fc2_weight_shape[1], k_sf_divisor), + ceil_div(fc2_weight_shape[0], 128), + 32, + 4, + 4, + ) + fc2_w_scales = fc2_w_scales.permute(3, 4, 1, 5, 2, 0) fc2_dactivation_kwargs["b_tensor"] = fc2_w_data fc2_dactivation_kwargs["sfb_tensor"] = fc2_w_scales else: @@ -3188,19 +3209,6 @@ def grouped_gemm_dactivation_kernel(cls) -> Callable: return grouped_gemm_dsrelu_wrapper_sm100 - @classmethod - @functools.lru_cache(maxsize=None) - def grouped_gemm_dactivation_is_deterministic(cls) -> bool: - """Feature-detect the dSReLU wrapper's ``deterministic`` argument (cuDNN FE 1.28.0+).""" - try: - kernel = cls.grouped_gemm_dactivation_kernel() - except ImportError: - return False - try: - return "deterministic" in inspect.signature(kernel).parameters - except (TypeError, ValueError): - return False - def fuse_glu_ops( ops: list[FusibleOperation], diff --git a/transformer_engine/pytorch/ops/fuser.py b/transformer_engine/pytorch/ops/fuser.py index 285f29ed54c..5c7371ce644 100644 --- a/transformer_engine/pytorch/ops/fuser.py +++ b/transformer_engine/pytorch/ops/fuser.py @@ -742,13 +742,6 @@ def _custom_ops_unsupported_reason( ) -> Optional[str]: """Why this pass may not run through its operations' custom ops.""" ops = self._forward_ops if mode == "forward" else self._backward_ops - if len(ops) != self._num_basic_ops or any( - op is not self._basic_ops[idx] or basic_op_idxs != [idx] - for idx, (op, basic_op_idxs) in enumerate(ops) - ): - return f"a {mode} fusion" - if self._num_basic_ops != 1: - return "a group of several operations" for op, kwargs in zip(self._basic_ops, basic_op_kwargs, strict=True): # Only tensors. The other fields of an args container are values read # off the module, constant across calls and baked into the graph; a @@ -761,6 +754,11 @@ def _custom_ops_unsupported_reason( for op in self._basic_ops: if op.num_extra_inputs or op.num_extra_outputs: return f"{type(op).__name__} with extra tensor inputs or outputs" + for op, basic_op_idxs in ops: + # A frozen prefix is not visited by backward, so its missing + # backward custom ops must not force the trainable suffix to eager. + if mode == "backward" and basic_op_idxs[-1] < self.first_op_requiring_backward: + continue reason = op.compile_unsupported_reason(mode) if reason is not None: return reason diff --git a/transformer_engine/pytorch/ops/op.py b/transformer_engine/pytorch/ops/op.py index 8d018e273a7..0bf247e12f1 100644 --- a/transformer_engine/pytorch/ops/op.py +++ b/transformer_engine/pytorch/ops/op.py @@ -8,6 +8,7 @@ import abc from collections.abc import Iterable, Sequence import dataclasses +import functools import pickle from typing import Any, Callable, Optional @@ -93,7 +94,7 @@ def _register_compile_ops(cls) -> None: register_custom_op( op_name=op_name, arg_type=arg_type, - impl=getattr(cls, f"{mode}_compute"), + impl=functools.partial(getattr(cls, f"{mode}_compute"), in_custom_op=True), fake_impl=getattr(cls, f"{mode}_compute_fake"), ) ) @@ -248,8 +249,12 @@ def fuser_backward( return grad_input, grad_params, grad_extra_inputs @classmethod - def forward_compute(cls, args: Any) -> tuple: - """Return (output, extra_outputs per basic op, fresh aux tensors).""" + def forward_compute(cls, args: Any, *, in_custom_op: bool = False) -> tuple: + """Return (output, extra_outputs per basic op, fresh aux tensors). + + The registered custom op passes in_custom_op=True; direct eager calls + use False. Inside a custom op, do not mutate tensors supplied in args. + """ raise NotImplementedError @classmethod @@ -258,10 +263,11 @@ def forward_compute_fake(cls, args: Any) -> tuple: raise NotImplementedError @classmethod - def backward_compute(cls, args: Any) -> tuple: + def backward_compute(cls, args: Any, *, in_custom_op: bool = False) -> tuple: """Return (grad_input, grad_params per basic op, grad_extra_inputs per basic op). A None grad_input passes grad_output through unchanged. + in_custom_op has the same meaning as in forward_compute. """ raise NotImplementedError @@ -715,7 +721,7 @@ def fuser_forward( ) if self.num_extra_inputs > 0 or self.num_extra_outputs > 0: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It should override `fuser_forward` instead of `op_forward`." @@ -750,7 +756,7 @@ def fuser_backward( ) if self.num_extra_inputs > 0 or self.num_extra_outputs > 0: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It should override `fuser_backward` instead of `op_backward`." diff --git a/transformer_engine/pytorch/quantized_tensor.py b/transformer_engine/pytorch/quantized_tensor.py index f6f2794bdf9..464305d25d1 100644 --- a/transformer_engine/pytorch/quantized_tensor.py +++ b/transformer_engine/pytorch/quantized_tensor.py @@ -338,7 +338,7 @@ def restore_from_saved( count = len(tensor.inner_names) inner = dict(zip(tensor.inner_names, saved_tensors[:count])) tensor_objects.append( - QuantizedTensorStorage.__tensor_unflatten__(inner, tensor.metadata, None, None) + tensor.metadata["cls"].__tensor_unflatten__(inner, tensor.metadata, None, None) ) saved_tensors = saved_tensors[count:] else: diff --git a/transformer_engine/pytorch/tensor/grouped_tensor.py b/transformer_engine/pytorch/tensor/grouped_tensor.py index 65b4a848228..b51d5209418 100644 --- a/transformer_engine/pytorch/tensor/grouped_tensor.py +++ b/transformer_engine/pytorch/tensor/grouped_tensor.py @@ -10,7 +10,7 @@ import torch from torch.utils._pytree import tree_map -from ..quantized_tensor import QuantizedTensorStorage, Quantizer +from ..quantized_tensor import QuantizedTensorStorage, Quantizer, _quantized_tensor_passthrough_ops from .storage.grouped_tensor_storage import GroupedTensorStorage from ..constants import DType @@ -182,6 +182,9 @@ def __torch_dispatch__(cls, func, types, args, kwargs=None): if kwargs is None: kwargs = {} + if func in _quantized_tensor_passthrough_ops: + return super().__torch_dispatch__(func, types, args, kwargs) + def copy_grouped_storage_metadata(dst: GroupedTensor, src: GroupedTensor) -> None: """Shallow-copy grouped-storage metadata onto wrapper outputs.""" dst.num_tensors = src.num_tensors diff --git a/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py index 6a48f85c6c7..f1bc3109d1c 100644 --- a/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py @@ -4,7 +4,7 @@ """Grouped tensor storage class for handling collections of tensors with different shapes""" from __future__ import annotations -from typing import Optional, Tuple, List, Union +from typing import Any, Dict, Optional, Tuple, List, Union import math import torch @@ -350,6 +350,61 @@ def nvfp4_e4m3_max(self) -> int: def nvfp4_e4m3_max(self, nvfp4_e4m3_max: int) -> None: self._nvfp4_e4m3_max = nvfp4_e4m3_max + _INNER_TENSORS = ( + ("rowwise_data", "data"), + ("columnwise_data", "columnwise_data"), + ("scale_inv", "scale_inv"), + ("columnwise_scale_inv", "columnwise_scale_inv"), + ("amax", "amax"), + ("columnwise_amax", "columnwise_amax"), + ("scale", "scale"), + ("first_dims", "first_dims"), + ("last_dims", "last_dims"), + ("tensor_offsets", "tensor_offsets"), + ) + + def __tensor_flatten__(self) -> Tuple[List[str], Dict[str, Any]]: + """Separate grouped buffers from their reconstruction metadata.""" + names = [name for name, _ in self._INNER_TENSORS if getattr(self, name) is not None] + shared_scale = self.scale_inv is not None and self.columnwise_scale_inv is self.scale_inv + if shared_scale: + names.remove("columnwise_scale_inv") + return names, { + "cls": type(self), + "is_tensor": isinstance(self, torch.Tensor), + "requires_grad": getattr(self, "requires_grad", False), + "shared_scale": shared_scale, + "nontensor_kwargs": { + "shape": self.logical_shape, + "dtype": self.fake_dtype, + "num_tensors": self.num_tensors, + "shapes": self.tensor_shapes, + "quantizer": self.quantizer, + "scale_inv_dtype": self._scale_inv_dtype, + "offsets": self.offsets, + "scale_inv_offsets": self.scale_inv_offsets, + "columnwise_scale_inv_offsets": self.columnwise_scale_inv_offsets, + "with_gemm_swizzled_scales": self._with_gemm_swizzled_scales, + "row_scaled_nvfp4": self.row_scaled_nvfp4, + "nvfp4_use_4over6": self.nvfp4_use_4over6, + "nvfp4_e4m3_max": self.nvfp4_e4m3_max, + }, + } + + @staticmethod + def __tensor_unflatten__(inner_tensors, metadata, outer_size, outer_stride): + """Rebuild a grouped storage or tensor without copying its buffers.""" + del outer_size + cls = metadata["cls"] + kwargs = dict(metadata["nontensor_kwargs"]) + kwargs.update({kwarg: inner_tensors.get(name) for name, kwarg in cls._INNER_TENSORS}) + if metadata.get("shared_scale", False): + kwargs["columnwise_scale_inv"] = kwargs["scale_inv"] + if metadata["is_tensor"]: + kwargs["requires_grad"] = metadata["requires_grad"] + kwargs["stride"] = outer_stride + return cls(**kwargs) + @property def scale_inv_dtype(self) -> Optional[DType]: """Data type of scale inverse buffers. diff --git a/transformer_engine/pytorch/triton/cross_entropy.py b/transformer_engine/pytorch/triton/cross_entropy.py index fda28474c28..3bc586c4ece 100644 --- a/transformer_engine/pytorch/triton/cross_entropy.py +++ b/transformer_engine/pytorch/triton/cross_entropy.py @@ -29,8 +29,13 @@ def cross_entropy_forward( dist_process_group: Union[dist.ProcessGroup, None], ignore_idx: int, overwrite_input: bool, + compute_argmax: bool = False, ): - """Forward implementation that saves the input and compact softmax statistics.""" + """Forward implementation that saves the input and compact softmax statistics. + + With ``compute_argmax``, also returns the index of each row's first maximum in the + full vocabulary (otherwise None). + """ B, SQ, V = _input.shape n_rows = B * SQ @@ -45,6 +50,9 @@ def cross_entropy_forward( loss_1d = torch.empty(n_rows, dtype=torch.float32, device=_input.device) stats = torch.empty((n_rows, 2), dtype=torch.float32, device=_input.device) n_non_ignore = torch.zeros(1, dtype=torch.int64, device=_input.device) if reduce_loss else None + argmax = ( + torch.empty(n_rows, dtype=torch.int64, device=_input.device) if compute_argmax else None + ) rank = 0 if dist_process_group is None else dist.get_rank(dist_process_group) world_size = 1 if dist_process_group is None else dist.get_world_size(dist_process_group) @@ -59,6 +67,7 @@ def cross_entropy_forward( Y_ptr=target, loss_ptr=loss_1d, stats_ptr=stats, + argmax_ptr=argmax if argmax is not None else stats, n_non_ignore=n_non_ignore if n_non_ignore is not None else stats, n_cols=V, n_rows_1=SQ, @@ -66,11 +75,17 @@ def cross_entropy_forward( label_smoothing=label_smoothing, COUNT_NON_IGNORE=reduce_loss, COPY_INPUT=not overwrite_input, + COMPUTE_ARGMAX=compute_argmax, BLOCK_SIZE=BLOCK_SIZE, num_warps=32, ) else: - local_data = torch.empty((n_rows, 4), dtype=torch.float32, device=_input.device) + # Per row and rank: max, sum of exponentials, target logit, logit sum, and with + # compute_argmax the local argmax (as FP32 bits) and its value (NaN if the shard has one) + local_data_size = 6 if compute_argmax else 4 + local_data = torch.empty( + (n_rows, local_data_size), dtype=torch.float32, device=_input.device + ) cross_entropy_tp_pre_kernel[(n_rows,)]( X_ptr=_input, X_stride_0=_input.stride(0), @@ -87,11 +102,13 @@ def cross_entropy_forward( COUNT_NON_IGNORE=reduce_loss, COMPUTE_X_SUM=label_smoothing > 0, COPY_INPUT=not overwrite_input, + COMPUTE_ARGMAX=compute_argmax, + LOCAL_DATA_SIZE=local_data_size, BLOCK_SIZE=BLOCK_SIZE, num_warps=32, ) gathered_data = torch.empty( - (world_size * n_rows, 4), dtype=torch.float32, device=_input.device + (world_size * n_rows, local_data_size), dtype=torch.float32, device=_input.device ) dist.all_gather_into_tensor(gathered_data, local_data, group=dist_process_group) cross_entropy_tp_post_kernel[(n_rows,)]( @@ -99,11 +116,14 @@ def cross_entropy_forward( Y_ptr=target, loss_ptr=loss_1d, stats_ptr=stats, + argmax_ptr=argmax if argmax is not None else stats, world_size=world_size, n_rows=n_rows, n_cols=V, ignore_idx=ignore_idx, label_smoothing=label_smoothing, + COMPUTE_ARGMAX=compute_argmax, + LOCAL_DATA_SIZE=local_data_size, num_warps=1, ) @@ -111,8 +131,10 @@ def cross_entropy_forward( if reduce_loss: n_non_ignore.clamp_(min=1) loss = loss_1d.sum() / n_non_ignore + if argmax is not None: + argmax = argmax.reshape(B, SQ) - return loss, saved_input, stats, target, n_non_ignore, rank, world_size + return loss, saved_input, stats, target, n_non_ignore, rank, world_size, argmax def cross_entropy_backward( diff --git a/transformer_engine/pytorch/utils.py b/transformer_engine/pytorch/utils.py index aa21ed06e05..648156aa965 100644 --- a/transformer_engine/pytorch/utils.py +++ b/transformer_engine/pytorch/utils.py @@ -740,6 +740,28 @@ def check_gemm_dims(inp: torch.Tensor, weight: torch.Tensor, fp8: bool) -> None: ) +def check_grouped_gemm_dims( + inp: torch.Tensor, + weight: torch.Tensor, + m_splits: Sequence[int], + fp8: bool, +) -> None: + """Check compiled grouped GEMM dimensions and host split sizes.""" + # pylint: disable=protected-access + check_gemm_dims(inp, weight, fp8) + torch._check( + math.prod(inp.shape[:-1]) == sum(m_splits), + lambda: "GEMM not possible: m_splits must sum to the input's token count", + ) + for split in m_splits: + torch._check(split >= 0, lambda: "m_splits entries must be non-negative") + if fp8: + torch._check( + all(m % 8 == 0 for m in m_splits), + lambda: "FP8 execution requires every m_splits entry to be divisible by 8", + ) + + def is_bf16_compatible() -> bool: """Replaces torch.cuda.is_bf16_compatible() with an explicit check on device compute capability to enforce sm_80 or higher.