From daddf0be6874e31948014e040c1412855df8ece9 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 8 Oct 2026 19:06:22 +0200 Subject: [PATCH 1/4] Move NCCL communicator lookup to Python Signed-off-by: Pawel Gadzinski --- build_tools/pytorch.py | 5 - docs/installation.rst | 5 + qa/L0_pytorch_unittest/test.sh | 1 + tests/pytorch/test_nccl_comm.py | 166 ++++++++++++++++++ transformer_engine/pytorch/csrc/extensions.h | 5 +- .../pytorch/csrc/extensions/comm.cpp | 68 ------- .../pytorch/csrc/extensions/ep.cpp | 2 +- .../pytorch/csrc/extensions/pybind.cpp | 5 - transformer_engine/pytorch/distributed.py | 31 ++++ transformer_engine/pytorch/ep.py | 6 +- .../pytorch/optimizers/newton_schulz.py | 19 +- 11 files changed, 210 insertions(+), 103 deletions(-) create mode 100644 tests/pytorch/test_nccl_comm.py delete mode 100644 transformer_engine/pytorch/csrc/extensions/comm.cpp diff --git a/build_tools/pytorch.py b/build_tools/pytorch.py index 173f9cee708..45bc04958d3 100644 --- a/build_tools/pytorch.py +++ b/build_tools/pytorch.py @@ -91,11 +91,6 @@ def setup_pytorch_extension( setup_mpi_flags(include_dirs, cxx_flags) - # ProcessGroupNCCL class declarations are gated on USE_C10D_NCCL. get_nccl_comm_ptr - # borrows the raw ncclComm_t from that backend and is used by both NCCL EP and the - # cuSolverMp path, so the macro is needed regardless of the NCCL EP gate below. - cxx_flags.append("-DUSE_C10D_NCCL") - # Mirror the NCCL EP gate from setup.py / common CMake. When disabled, the # ep.cpp source no-ops at the #ifdef boundary; without the define it would # produce undefined references to nvte_ep_*. diff --git a/docs/installation.rst b/docs/installation.rst index 9d3eada827c..2abcdfdf73b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -55,6 +55,11 @@ Additional Prerequisites 1. [For PyTorch support] `PyTorch `__ with GPU support. 2. [For JAX support] `JAX `__ with GPU support, version >= 0.4.7. +PyTorch EP and cuSolverMp require the selected NCCL backend to expose its +communicator through Python's ``comm_ptr`` property or ``_comm_ptr()`` method. +Some PyTorch builds expose these only for the legacy NCCL backend; using NCCL2 +or NCCL lazy with these features requires a build that also exposes its getter. + Installation (stable release) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 2a486450a20..884da26824d 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -30,6 +30,7 @@ export NVTE_FLASH_ATTN_V4=0 pip3 install pytest==8.2.1 || error_exit "Failed to install pytest" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_optional_flash_attn_import.xml $TE_PATH/tests/pytorch/test_optional_flash_attn_import.py || test_fail "test_optional_flash_attn_import.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_nccl_comm.xml $TE_PATH/tests/pytorch/test_nccl_comm.py || test_fail "test_nccl_comm.py" NVTE_GROUPED_LINEAR_SINGLE_PARAM=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/test_sanity.py || test_fail "test_sanity.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_recipe.xml $TE_PATH/tests/pytorch/test_recipe.py || test_fail "test_recipe.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_custom_recipe.xml $TE_PATH/tests/pytorch/test_custom_recipe.py || test_fail "test_custom_recipe.py" diff --git a/tests/pytorch/test_nccl_comm.py b/tests/pytorch/test_nccl_comm.py new file mode 100644 index 00000000000..7f1c86a337f --- /dev/null +++ b/tests/pytorch/test_nccl_comm.py @@ -0,0 +1,166 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""NCCL communicator borrowing across PyTorch backend interfaces.""" + +import ctypes +from datetime import timedelta +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from transformer_engine.pytorch.distributed import get_nccl_comm_ptr + + +@pytest.fixture +def borrow_env(monkeypatch): + monkeypatch.setattr(torch.cuda, "current_device", lambda: 3) + barrier = Mock() + monkeypatch.setattr(dist, "barrier", barrier) + return barrier + + +@pytest.mark.parametrize("interface", ["comm_ptr", "_comm_ptr"]) +@pytest.mark.parametrize("backend_name", ["nccl", "nccl2", "nccl-lazy"]) +@pytest.mark.parametrize("wrapped", [False, True]) +def test_borrow_nccl_comm(borrow_env, interface, backend_name, wrapped): + pointer = 0x123456789ABC + group = Mock() + + def get_pointer(_self=None): + borrow_env.assert_called_once_with(group=group, device_ids=[3]) + return pointer + + getter = property(get_pointer) if interface == "comm_ptr" else get_pointer + backend = type("Backend", (), {interface: getter, "name": lambda _: backend_name})() + if wrapped: + backend = SimpleNamespace(wrapped_pg=SimpleNamespace(wrapped_pg=backend)) + group._get_backend.return_value = backend + assert get_nccl_comm_ptr(group) == pointer + group._get_backend.assert_called_once_with(torch.device("cuda", 3)) + + +@pytest.mark.parametrize("pointer", [0, -1, None, True, "123", 1.5]) +def test_invalid_nccl_comm(borrow_env, pointer): + backend = SimpleNamespace(name=lambda: "nccl", comm_ptr=pointer, _comm_ptr=Mock()) + group = Mock() + group._get_backend.return_value = backend + with pytest.raises(RuntimeError, match="invalid communicator pointer"): + get_nccl_comm_ptr(group) + backend._comm_ptr.assert_not_called() + + +def test_missing_nccl_getter(borrow_env): + group = Mock() + group._get_backend.return_value = SimpleNamespace(name=lambda: "nccl2") + with pytest.raises(RuntimeError, match="does not expose a NCCL communicator pointer"): + get_nccl_comm_ptr(group) + + +def test_non_nccl_backend(borrow_env): + group = Mock() + group._get_backend.return_value = SimpleNamespace(name=lambda: "gloo", comm_ptr=123) + with pytest.raises(RuntimeError, match="Expected a NCCL CUDA backend"): + get_nccl_comm_ptr(group) + borrow_env.assert_not_called() + + +def test_getter_error_propagates(borrow_env): + class Backend: + name = lambda _: "nccl" + _comm_ptr = Mock() + + @property + def comm_ptr(self): + raise RuntimeError("communicator aborted") + + group = Mock() + group._get_backend.return_value = Backend() + with pytest.raises(RuntimeError, match="communicator aborted"): + get_nccl_comm_ptr(group) + Backend._comm_ptr.assert_not_called() + + +def _run_nccl_rank(rank, world_size, store_path, debug, legacy): + torch.cuda.set_device(rank) + dist.set_debug_level(dist.DebugLevel.DETAIL if debug else dist.DebugLevel.OFF) + backend_name = "nccl-legacy" if legacy and hasattr(dist, "ProcessGroupNCCL2") else "nccl" + dist.init_process_group( + backend_name, + init_method=f"file://{store_path}", + rank=rank, + world_size=world_size, + timeout=timedelta(seconds=60), + ) + try: + group = dist.group.WORLD + backend = group._get_backend(torch.device("cuda", rank)) + if debug: + assert hasattr(backend, "wrapped_pg") + backend = backend.wrapped_pg + if not hasattr(type(backend), "comm_ptr") and not hasattr(backend, "_comm_ptr"): + with pytest.raises(RuntimeError, match="does not expose a NCCL communicator pointer"): + get_nccl_comm_ptr(group) + return + + comm = ctypes.c_void_p(get_nccl_comm_ptr(group)) + nccl = ctypes.CDLL("libnccl.so.2") + for name, expected in [ + ("ncclCommCount", world_size), + ("ncclCommUserRank", rank), + ("ncclCommCuDevice", rank), + ]: + query = getattr(nccl, name) + query.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)] + query.restype = ctypes.c_int + value = ctypes.c_int() + assert query(comm, ctypes.byref(value)) == 0 + assert value.value == expected + + nccl.ncclAllReduce.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_int, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_void_p, + ] + nccl.ncclAllReduce.restype = ctypes.c_int + data = torch.full((32,), rank + 1, dtype=torch.float32, device="cuda") + stream = torch.cuda.current_stream().cuda_stream + # NCCL float32 = 7, sum = 0. + assert nccl.ncclAllReduce(data.data_ptr(), data.data_ptr(), 32, 7, 0, comm, stream) == 0 + torch.testing.assert_close(data, torch.full_like(data, world_size * (world_size + 1) / 2)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + stream = torch.cuda.current_stream().cuda_stream + assert nccl.ncclAllReduce(data.data_ptr(), data.data_ptr(), 32, 7, 0, comm, stream) == 0 + for _ in range(3): + data.fill_(rank + 1) + graph.replay() + torch.testing.assert_close( + data, torch.full_like(data, world_size * (world_size + 1) / 2) + ) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("world_size", [1, 2]) +@pytest.mark.parametrize("debug", [False, True]) +@pytest.mark.parametrize("legacy", [False, True]) +def test_native_nccl_comm(tmp_path, world_size, debug, legacy): + if not dist.is_nccl_available() or torch.cuda.device_count() < world_size: + pytest.skip(f"Requires NCCL and {world_size} CUDA devices") + if debug and not dist.is_gloo_available(): + pytest.skip("DEBUG DETAIL requires Gloo") + mp.spawn( + _run_nccl_rank, + args=(world_size, str(tmp_path / "store"), debug, legacy), + nprocs=world_size, + ) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 955763f43b1..1307ebb3625 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -736,10 +736,7 @@ void grouped_swizzle_for_gemm(py::handle &tensor, bool rowwise, bool columnwise) * Expert Parallelism **************************************************************************************************/ -// Borrow torch's host ncclComm_t from a process group's CUDA backend. -int64_t get_nccl_comm_ptr(c10d::ProcessGroup *process_group); - -// Borrows torch's NCCL host comm (``comm_ptr`` from ``get_nccl_comm_ptr``). +// Borrows torch's NCCL host comm passed from Python. // ``group_name`` is the PG name used by the symm-mem window resolver. // ``zero_copy`` is forwarded into ``NVTEEpGroupConfig.zero_copy``. void ep_initialize(uintptr_t comm_ptr, const std::string &group_name, int64_t num_experts, diff --git a/transformer_engine/pytorch/csrc/extensions/comm.cpp b/transformer_engine/pytorch/csrc/extensions/comm.cpp deleted file mode 100644 index ebdddb2dd7e..00000000000 --- a/transformer_engine/pytorch/csrc/extensions/comm.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/************************************************************************* - * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * - * See LICENSE for license information. - ************************************************************************/ - -// NCCL backend headers, for borrowing the raw ncclComm_t. The NCCL2 backend -// header exists only on torch builds that ship it. -#include -#if __has_include() -#include -#define NVTE_HAS_NCCL2_PG 1 -#endif - -#include -#include - -#include "../common.h" -#include "../extensions.h" - -namespace transformer_engine::pytorch { - -namespace { - -// ProcessGroupNCCL::getCommPtr() was added in torch 2.8; older backends do not -// expose the raw communicator. Detect it so the borrow compiles on torch >= 2.1 -// and fails only at runtime when the backend cannot provide the communicator. -template -struct has_get_comm_ptr : std::false_type {}; -template -struct has_get_comm_ptr().getCommPtr())>> - : std::true_type {}; - -template -int64_t borrow_comm_ptr(PG* pg) { - if constexpr (has_get_comm_ptr::value) { - return pg->getCommPtr(); - } else { - NVTE_ERROR( - "get_nccl_comm_ptr: this PyTorch version does not expose the raw NCCL communicator " - "(ProcessGroupNCCL::getCommPtr()); torch >= 2.8 is required to borrow it for EP or " - "cuSolverMp."); - } -} - -} // namespace - -int64_t get_nccl_comm_ptr(c10d::ProcessGroup* process_group) { - NVTE_CHECK(process_group != nullptr, "get_nccl_comm_ptr: process_group must be non-null"); - auto backend = process_group->getBackend(c10::DeviceType::CUDA); - NVTE_CHECK(backend, "get_nccl_comm_ptr: process group has no CUDA backend"); - - // Classic backend (ProcessGroupNCCL._comm_ptr() in Python). - if (auto* nccl_pg = dynamic_cast(backend.get())) { - return borrow_comm_ptr(nccl_pg); - } - // NCCL2 backend, default on recent torch. -#ifdef NVTE_HAS_NCCL2_PG - if (auto* nccl2_pg = dynamic_cast(backend.get())) { - return borrow_comm_ptr(nccl2_pg); - } -#endif - - NVTE_ERROR("get_nccl_comm_ptr requires a NCCL-backed process group, but backend '", - backend->getBackendName(), "' does not expose a borrowable NCCL communicator"); -} - -} // namespace transformer_engine::pytorch diff --git a/transformer_engine/pytorch/csrc/extensions/ep.cpp b/transformer_engine/pytorch/csrc/extensions/ep.cpp index 66316a07ccd..65391c1121b 100644 --- a/transformer_engine/pytorch/csrc/extensions/ep.cpp +++ b/transformer_engine/pytorch/csrc/extensions/ep.cpp @@ -180,7 +180,7 @@ bool ep_zero_copy_supported() { } // ── Bootstrap ──────────────────────────────────────────────────────────────── -// ``comm_ptr`` is torch's borrowed NCCL host comm (see get_nccl_comm_ptr). +// ``comm_ptr`` is torch's borrowed NCCL host comm passed from Python. // ``group_name`` is captured for the symm-mem window resolver. void ep_initialize(uintptr_t comm_ptr, const std::string& group_name, int64_t num_experts, diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 2046408eba8..4e04ee806fa 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -714,11 +714,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { &transformer_engine::pytorch::multi_tensor_compute_scale_inv_e8m0_cuda, "Fused compute E8M0 scale_inv from amax", py::call_guard()); - // Borrow torch's host ncclComm_t from a process group (used by NCCL EP and cuSolverMp). - m.def("get_nccl_comm_ptr", &transformer_engine::pytorch::get_nccl_comm_ptr, - "Borrow torch's host ncclComm_t from a process group's CUDA backend.", - py::arg("process_group")); - // Newton-Schulz (cuSolverMp) m.def("cusolvermp_ctx_create", &transformer_engine::pytorch::cusolvermp_ctx_create, "Create cuSolverMp context for Newton-Schulz", py::arg("nccl_comm_ptr"), py::arg("nranks"), diff --git a/transformer_engine/pytorch/distributed.py b/transformer_engine/pytorch/distributed.py index d5802ccb61f..0d8a8a7c670 100644 --- a/transformer_engine/pytorch/distributed.py +++ b/transformer_engine/pytorch/distributed.py @@ -54,6 +54,37 @@ __all__ = ["checkpoint", "CudaRNGStatesTracker"] +def get_nccl_comm_ptr(group: torch.distributed.ProcessGroup) -> int: + """Materialize and borrow the current device's NCCL communicator. + + This is collective over ``group``. The caller must keep the group alive + until all users of the borrowed communicator have been destroyed. + """ + device = torch.cuda.current_device() + backend = group._get_backend(torch.device("cuda", device)) + while hasattr(backend, "wrapped_pg"): + backend = backend.wrapped_pg + if backend.name() not in ("nccl", "nccl2", "nccl-lazy"): + raise RuntimeError(f"Expected a NCCL CUDA backend, got '{backend.name()}'") + + # The communicator must exist on this device before either getter is called. + torch.distributed.barrier(group=group, device_ids=[device]) + try: + comm_ptr = backend.comm_ptr + except AttributeError: + getter = getattr(backend, "_comm_ptr", None) + if getter is None: + raise RuntimeError( + f"PyTorch {torch.__version__} backend '{backend.name()}' does not expose " + "a NCCL communicator pointer. Use a PyTorch build with comm_ptr or " + "_comm_ptr support for this backend." + ) from None + comm_ptr = getter() + if not isinstance(comm_ptr, int) or isinstance(comm_ptr, bool) or comm_ptr <= 0: + raise RuntimeError("NCCL backend returned an invalid communicator pointer") + return comm_ptr + + _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = { "tensor_model_parallel": False, "partition_dim": -1, diff --git a/transformer_engine/pytorch/ep.py b/transformer_engine/pytorch/ep.py index 997f15561e5..e37cc580dfd 100644 --- a/transformer_engine/pytorch/ep.py +++ b/transformer_engine/pytorch/ep.py @@ -17,7 +17,7 @@ from .constants import DType from .cpu_offload import mark_not_offload -from .distributed import symm_mem_alloc, release_symm_mem_pool +from .distributed import get_nccl_comm_ptr, symm_mem_alloc, release_symm_mem_pool from .quantized_tensor import QuantizedTensor, QuantizedTensorStorage from .tensor.storage.mxfp8_tensor_storage import MXFP8TensorStorage @@ -194,9 +194,7 @@ def ep_bootstrap( stacklevel=2, ) - # Materialize the PG's NCCL comm before borrowing its raw handle. - dist.barrier(group=ep_group, device_ids=[torch.cuda.current_device()]) - comm_ptr = tex.get_nccl_comm_ptr(ep_group) + comm_ptr = get_nccl_comm_ptr(ep_group) tex.ep_initialize( int(comm_ptr), diff --git a/transformer_engine/pytorch/optimizers/newton_schulz.py b/transformer_engine/pytorch/optimizers/newton_schulz.py index e1b1a0bcbde..a7843e580a4 100644 --- a/transformer_engine/pytorch/optimizers/newton_schulz.py +++ b/transformer_engine/pytorch/optimizers/newton_schulz.py @@ -12,6 +12,8 @@ import transformer_engine_torch as tex +from ..distributed import get_nccl_comm_ptr + _COEFFICIENT_SETS = { # Values are rounded to closest representable in single precision. @@ -148,7 +150,7 @@ def __init__(self, group: dist.ProcessGroup) -> None: if self.rank < 0: raise RuntimeError("The current process is not a member of the supplied process group") - comm_ptr = _get_nccl_comm_ptr(group) + comm_ptr = get_nccl_comm_ptr(group) self._ptr = tex.cusolvermp_ctx_create(comm_ptr, self.nranks, self.rank) @property @@ -172,21 +174,6 @@ def __del__(self) -> None: self.destroy() -def _get_nccl_comm_ptr(group: dist.ProcessGroup) -> int: - """Materialize and borrow a raw NCCL communicator from a process group.""" - backend = dist.get_backend(group) - if backend != "nccl": - raise RuntimeError(f"Newton-Schulz requires NCCL backend, got '{backend}'") - - # The NCCL backend creates communicators lazily; this collective materializes - # the borrowed communicator on every rank first. - dist.barrier(group=group, device_ids=[torch.cuda.current_device()]) - comm_ptr = tex.get_nccl_comm_ptr(group) - if not isinstance(comm_ptr, int) or comm_ptr == 0: - raise RuntimeError("NCCL backend returned an invalid communicator pointer") - return comm_ptr - - def newton_schulz( x: torch.Tensor, ctx: CusolverMpCtx, From a5e58a6585e1f97122ad4c5859d39d976f9487de Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 8 Oct 2026 19:18:09 +0200 Subject: [PATCH 2/4] Fix device binding and graph teardown in NCCL tests Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_nccl_comm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pytorch/test_nccl_comm.py b/tests/pytorch/test_nccl_comm.py index 7f1c86a337f..54ff9be1f61 100644 --- a/tests/pytorch/test_nccl_comm.py +++ b/tests/pytorch/test_nccl_comm.py @@ -95,8 +95,10 @@ def _run_nccl_rank(rank, world_size, store_path, debug, legacy): init_method=f"file://{store_path}", rank=rank, world_size=world_size, + device_id=torch.device("cuda", rank), timeout=timedelta(seconds=60), ) + graph = None try: group = dist.group.WORLD backend = group._get_backend(torch.device("cuda", rank)) @@ -148,6 +150,7 @@ def _run_nccl_rank(rank, world_size, store_path, debug, legacy): data, torch.full_like(data, world_size * (world_size + 1) / 2) ) finally: + del graph dist.destroy_process_group() From 80d73c1018bab7418de263dc712ec9fca6bde23b Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 8 Oct 2026 20:30:24 +0200 Subject: [PATCH 3/4] Keep a C++ NCCL fallback for older PyTorch builds Signed-off-by: Pawel Gadzinski --- build_tools/pytorch.py | 9 +++ docs/installation.rst | 10 +-- tests/pytorch/test_nccl_comm.py | 60 +++++++++++++--- transformer_engine/pytorch/csrc/extensions.h | 4 ++ .../pytorch/csrc/extensions/comm.cpp | 70 +++++++++++++++++++ .../pytorch/csrc/extensions/pybind.cpp | 5 ++ transformer_engine/pytorch/distributed.py | 7 +- 7 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 transformer_engine/pytorch/csrc/extensions/comm.cpp diff --git a/build_tools/pytorch.py b/build_tools/pytorch.py index 45bc04958d3..245e922a9c9 100644 --- a/build_tools/pytorch.py +++ b/build_tools/pytorch.py @@ -9,6 +9,7 @@ from importlib import metadata import setuptools +from packaging.version import Version from .utils import ( all_files_in_dir, @@ -91,6 +92,14 @@ def setup_pytorch_extension( setup_mpi_flags(include_dirs, cxx_flags) + # Older NVIDIA snapshots also need this fallback despite reporting torch 2.15. + from torch import __version__ as torch_version + + if Version(torch_version).release < (2, 16) and not bool( + int(os.getenv("NVTE_TORCH_STABLE_ABI", "0")) + ): + cxx_flags.extend(["-DNVTE_WITH_TORCH_NCCL_COMM", "-DUSE_C10D_NCCL"]) + # Mirror the NCCL EP gate from setup.py / common CMake. When disabled, the # ep.cpp source no-ops at the #ifdef boundary; without the define it would # produce undefined references to nvte_ep_*. diff --git a/docs/installation.rst b/docs/installation.rst index 2abcdfdf73b..513a49b888d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -55,10 +55,12 @@ Additional Prerequisites 1. [For PyTorch support] `PyTorch `__ with GPU support. 2. [For JAX support] `JAX `__ with GPU support, version >= 0.4.7. -PyTorch EP and cuSolverMp require the selected NCCL backend to expose its -communicator through Python's ``comm_ptr`` property or ``_comm_ptr()`` method. -Some PyTorch builds expose these only for the legacy NCCL backend; using NCCL2 -or NCCL lazy with these features requires a build that also exposes its getter. +PyTorch EP and cuSolverMp use the selected NCCL backend's Python ``comm_ptr`` +property or ``_comm_ptr()`` method when available. Builds against PyTorch older +than 2.16 retain a C++ fallback for backends without either Python getter, +including older NVIDIA PyTorch 2.15 snapshots. This fallback is excluded when +building with PyTorch 2.16 or newer, or with ``NVTE_TORCH_STABLE_ABI=1``; those +builds require the backend's Python getter. Installation (stable release) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/pytorch/test_nccl_comm.py b/tests/pytorch/test_nccl_comm.py index 54ff9be1f61..197c10eb059 100644 --- a/tests/pytorch/test_nccl_comm.py +++ b/tests/pytorch/test_nccl_comm.py @@ -15,6 +15,7 @@ import torch.multiprocessing as mp from transformer_engine.pytorch.distributed import get_nccl_comm_ptr +import transformer_engine_torch as tex @pytest.fixture @@ -22,6 +23,12 @@ def borrow_env(monkeypatch): monkeypatch.setattr(torch.cuda, "current_device", lambda: 3) barrier = Mock() monkeypatch.setattr(dist, "barrier", barrier) + monkeypatch.setattr( + tex, + "get_nccl_comm_ptr", + Mock(side_effect=AssertionError("Unexpected C++ fallback")), + raising=False, + ) return barrier @@ -43,19 +50,48 @@ def get_pointer(_self=None): group._get_backend.return_value = backend assert get_nccl_comm_ptr(group) == pointer group._get_backend.assert_called_once_with(torch.device("cuda", 3)) + tex.get_nccl_comm_ptr.assert_not_called() @pytest.mark.parametrize("pointer", [0, -1, None, True, "123", 1.5]) -def test_invalid_nccl_comm(borrow_env, pointer): - backend = SimpleNamespace(name=lambda: "nccl", comm_ptr=pointer, _comm_ptr=Mock()) +@pytest.mark.parametrize("native", [False, True]) +def test_invalid_nccl_comm(borrow_env, pointer, native): + backend = SimpleNamespace(name=lambda: "nccl") + if native: + tex.get_nccl_comm_ptr.side_effect = None + tex.get_nccl_comm_ptr.return_value = pointer + else: + backend.comm_ptr = pointer + backend._comm_ptr = Mock() group = Mock() group._get_backend.return_value = backend with pytest.raises(RuntimeError, match="invalid communicator pointer"): get_nccl_comm_ptr(group) - backend._comm_ptr.assert_not_called() + if native: + tex.get_nccl_comm_ptr.assert_called_once_with(backend) + else: + backend._comm_ptr.assert_not_called() + tex.get_nccl_comm_ptr.assert_not_called() + + +@pytest.mark.parametrize("wrapped", [False, True]) +def test_cpp_fallback(borrow_env, wrapped): + backend = SimpleNamespace(name=lambda: "nccl2") + group = Mock() + group._get_backend.return_value = SimpleNamespace(wrapped_pg=backend) if wrapped else backend + + def get_pointer(actual_backend): + assert actual_backend is backend + borrow_env.assert_called_once_with(group=group, device_ids=[3]) + return 123 + + tex.get_nccl_comm_ptr.side_effect = get_pointer + assert get_nccl_comm_ptr(group) == 123 + tex.get_nccl_comm_ptr.assert_called_once_with(backend) -def test_missing_nccl_getter(borrow_env): +def test_missing_nccl_getter(borrow_env, monkeypatch): + monkeypatch.delattr(tex, "get_nccl_comm_ptr") group = Mock() group._get_backend.return_value = SimpleNamespace(name=lambda: "nccl2") with pytest.raises(RuntimeError, match="does not expose a NCCL communicator pointer"): @@ -84,6 +120,17 @@ def comm_ptr(self): with pytest.raises(RuntimeError, match="communicator aborted"): get_nccl_comm_ptr(group) Backend._comm_ptr.assert_not_called() + tex.get_nccl_comm_ptr.assert_not_called() + + +def test_cpp_getter_error_propagates(borrow_env): + group = Mock() + backend = SimpleNamespace(name=lambda: "nccl2") + group._get_backend.return_value = backend + tex.get_nccl_comm_ptr.side_effect = RuntimeError("communicator aborted") + with pytest.raises(RuntimeError, match="communicator aborted"): + get_nccl_comm_ptr(group) + tex.get_nccl_comm_ptr.assert_called_once_with(backend) def _run_nccl_rank(rank, world_size, store_path, debug, legacy): @@ -105,11 +152,6 @@ def _run_nccl_rank(rank, world_size, store_path, debug, legacy): if debug: assert hasattr(backend, "wrapped_pg") backend = backend.wrapped_pg - if not hasattr(type(backend), "comm_ptr") and not hasattr(backend, "_comm_ptr"): - with pytest.raises(RuntimeError, match="does not expose a NCCL communicator pointer"): - get_nccl_comm_ptr(group) - return - comm = ctypes.c_void_p(get_nccl_comm_ptr(group)) nccl = ctypes.CDLL("libnccl.so.2") for name, expected in [ diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 1307ebb3625..4ced9d67848 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -736,6 +736,10 @@ void grouped_swizzle_for_gemm(py::handle &tensor, bool rowwise, bool columnwise) * Expert Parallelism **************************************************************************************************/ +#ifdef NVTE_WITH_TORCH_NCCL_COMM +int64_t get_nccl_comm_ptr(c10d::Backend *backend); +#endif + // Borrows torch's NCCL host comm passed from Python. // ``group_name`` is the PG name used by the symm-mem window resolver. // ``zero_copy`` is forwarded into ``NVTEEpGroupConfig.zero_copy``. diff --git a/transformer_engine/pytorch/csrc/extensions/comm.cpp b/transformer_engine/pytorch/csrc/extensions/comm.cpp new file mode 100644 index 00000000000..206bf99fdf1 --- /dev/null +++ b/transformer_engine/pytorch/csrc/extensions/comm.cpp @@ -0,0 +1,70 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#ifdef NVTE_WITH_TORCH_NCCL_COMM + +// NCCL backend headers, for borrowing the raw ncclComm_t. The NCCL2 backend +// header exists only on torch builds that ship it. +#include +#if __has_include() +#include +#define NVTE_HAS_NCCL2_PG 1 +#endif + +#include +#include + +#include "../common.h" +#include "../extensions.h" + +namespace transformer_engine::pytorch { + +namespace { + +// ProcessGroupNCCL::getCommPtr() was added in torch 2.8; older backends do not +// expose the raw communicator. Detect it so the borrow compiles on torch >= 2.1 +// and fails only at runtime when the backend cannot provide the communicator. +template +struct has_get_comm_ptr : std::false_type {}; +template +struct has_get_comm_ptr().getCommPtr())>> + : std::true_type {}; + +template +int64_t borrow_comm_ptr(PG* pg) { + if constexpr (has_get_comm_ptr::value) { + return pg->getCommPtr(); + } else { + NVTE_ERROR( + "get_nccl_comm_ptr: this PyTorch version does not expose the raw NCCL communicator " + "(ProcessGroupNCCL::getCommPtr()); torch >= 2.8 is required to borrow it for EP or " + "cuSolverMp."); + } +} + +} // namespace + +int64_t get_nccl_comm_ptr(c10d::Backend* backend) { + NVTE_CHECK(backend != nullptr, "get_nccl_comm_ptr: backend must be non-null"); + + // Classic backend (ProcessGroupNCCL._comm_ptr() in Python). + if (auto* nccl_pg = dynamic_cast(backend)) { + return borrow_comm_ptr(nccl_pg); + } + // NCCL2 backend, default on recent torch. +#ifdef NVTE_HAS_NCCL2_PG + if (auto* nccl2_pg = dynamic_cast(backend)) { + return borrow_comm_ptr(nccl2_pg); + } +#endif + + NVTE_ERROR("get_nccl_comm_ptr requires a NCCL-backed process group, but backend '", + backend->getBackendName(), "' does not expose a borrowable NCCL communicator"); +} + +} // namespace transformer_engine::pytorch + +#endif // NVTE_WITH_TORCH_NCCL_COMM diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 4e04ee806fa..1252285e834 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -714,6 +714,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { &transformer_engine::pytorch::multi_tensor_compute_scale_inv_e8m0_cuda, "Fused compute E8M0 scale_inv from amax", py::call_guard()); +#ifdef NVTE_WITH_TORCH_NCCL_COMM + m.def("get_nccl_comm_ptr", &transformer_engine::pytorch::get_nccl_comm_ptr, + "Borrow torch's host ncclComm_t from a CUDA backend.", py::arg("backend")); +#endif + // Newton-Schulz (cuSolverMp) m.def("cusolvermp_ctx_create", &transformer_engine::pytorch::cusolvermp_ctx_create, "Create cuSolverMp context for Newton-Schulz", py::arg("nccl_comm_ptr"), py::arg("nranks"), diff --git a/transformer_engine/pytorch/distributed.py b/transformer_engine/pytorch/distributed.py index 0d8a8a7c670..7f4674e9fb7 100644 --- a/transformer_engine/pytorch/distributed.py +++ b/transformer_engine/pytorch/distributed.py @@ -73,13 +73,16 @@ def get_nccl_comm_ptr(group: torch.distributed.ProcessGroup) -> int: comm_ptr = backend.comm_ptr except AttributeError: getter = getattr(backend, "_comm_ptr", None) - if getter is None: + if getter is not None: + comm_ptr = getter() + elif hasattr(tex, "get_nccl_comm_ptr"): + comm_ptr = tex.get_nccl_comm_ptr(backend) + else: raise RuntimeError( f"PyTorch {torch.__version__} backend '{backend.name()}' does not expose " "a NCCL communicator pointer. Use a PyTorch build with comm_ptr or " "_comm_ptr support for this backend." ) from None - comm_ptr = getter() if not isinstance(comm_ptr, int) or isinstance(comm_ptr, bool) or comm_ptr <= 0: raise RuntimeError("NCCL backend returned an invalid communicator pointer") return comm_ptr From 1e2a6168a7397d41a2324e8a5e478df18cf9753a Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 8 Oct 2026 20:54:47 +0200 Subject: [PATCH 4/4] Rely on existing EP and cuSolverMp integration tests Signed-off-by: Pawel Gadzinski --- qa/L0_pytorch_unittest/test.sh | 1 - tests/pytorch/test_nccl_comm.py | 211 -------------------------------- 2 files changed, 212 deletions(-) delete mode 100644 tests/pytorch/test_nccl_comm.py diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 884da26824d..2a486450a20 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -30,7 +30,6 @@ export NVTE_FLASH_ATTN_V4=0 pip3 install pytest==8.2.1 || error_exit "Failed to install pytest" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_optional_flash_attn_import.xml $TE_PATH/tests/pytorch/test_optional_flash_attn_import.py || test_fail "test_optional_flash_attn_import.py" -python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_nccl_comm.xml $TE_PATH/tests/pytorch/test_nccl_comm.py || test_fail "test_nccl_comm.py" NVTE_GROUPED_LINEAR_SINGLE_PARAM=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/test_sanity.py || test_fail "test_sanity.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_recipe.xml $TE_PATH/tests/pytorch/test_recipe.py || test_fail "test_recipe.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_custom_recipe.xml $TE_PATH/tests/pytorch/test_custom_recipe.py || test_fail "test_custom_recipe.py" diff --git a/tests/pytorch/test_nccl_comm.py b/tests/pytorch/test_nccl_comm.py deleted file mode 100644 index 197c10eb059..00000000000 --- a/tests/pytorch/test_nccl_comm.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. - -"""NCCL communicator borrowing across PyTorch backend interfaces.""" - -import ctypes -from datetime import timedelta -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest -import torch -import torch.distributed as dist -import torch.multiprocessing as mp - -from transformer_engine.pytorch.distributed import get_nccl_comm_ptr -import transformer_engine_torch as tex - - -@pytest.fixture -def borrow_env(monkeypatch): - monkeypatch.setattr(torch.cuda, "current_device", lambda: 3) - barrier = Mock() - monkeypatch.setattr(dist, "barrier", barrier) - monkeypatch.setattr( - tex, - "get_nccl_comm_ptr", - Mock(side_effect=AssertionError("Unexpected C++ fallback")), - raising=False, - ) - return barrier - - -@pytest.mark.parametrize("interface", ["comm_ptr", "_comm_ptr"]) -@pytest.mark.parametrize("backend_name", ["nccl", "nccl2", "nccl-lazy"]) -@pytest.mark.parametrize("wrapped", [False, True]) -def test_borrow_nccl_comm(borrow_env, interface, backend_name, wrapped): - pointer = 0x123456789ABC - group = Mock() - - def get_pointer(_self=None): - borrow_env.assert_called_once_with(group=group, device_ids=[3]) - return pointer - - getter = property(get_pointer) if interface == "comm_ptr" else get_pointer - backend = type("Backend", (), {interface: getter, "name": lambda _: backend_name})() - if wrapped: - backend = SimpleNamespace(wrapped_pg=SimpleNamespace(wrapped_pg=backend)) - group._get_backend.return_value = backend - assert get_nccl_comm_ptr(group) == pointer - group._get_backend.assert_called_once_with(torch.device("cuda", 3)) - tex.get_nccl_comm_ptr.assert_not_called() - - -@pytest.mark.parametrize("pointer", [0, -1, None, True, "123", 1.5]) -@pytest.mark.parametrize("native", [False, True]) -def test_invalid_nccl_comm(borrow_env, pointer, native): - backend = SimpleNamespace(name=lambda: "nccl") - if native: - tex.get_nccl_comm_ptr.side_effect = None - tex.get_nccl_comm_ptr.return_value = pointer - else: - backend.comm_ptr = pointer - backend._comm_ptr = Mock() - group = Mock() - group._get_backend.return_value = backend - with pytest.raises(RuntimeError, match="invalid communicator pointer"): - get_nccl_comm_ptr(group) - if native: - tex.get_nccl_comm_ptr.assert_called_once_with(backend) - else: - backend._comm_ptr.assert_not_called() - tex.get_nccl_comm_ptr.assert_not_called() - - -@pytest.mark.parametrize("wrapped", [False, True]) -def test_cpp_fallback(borrow_env, wrapped): - backend = SimpleNamespace(name=lambda: "nccl2") - group = Mock() - group._get_backend.return_value = SimpleNamespace(wrapped_pg=backend) if wrapped else backend - - def get_pointer(actual_backend): - assert actual_backend is backend - borrow_env.assert_called_once_with(group=group, device_ids=[3]) - return 123 - - tex.get_nccl_comm_ptr.side_effect = get_pointer - assert get_nccl_comm_ptr(group) == 123 - tex.get_nccl_comm_ptr.assert_called_once_with(backend) - - -def test_missing_nccl_getter(borrow_env, monkeypatch): - monkeypatch.delattr(tex, "get_nccl_comm_ptr") - group = Mock() - group._get_backend.return_value = SimpleNamespace(name=lambda: "nccl2") - with pytest.raises(RuntimeError, match="does not expose a NCCL communicator pointer"): - get_nccl_comm_ptr(group) - - -def test_non_nccl_backend(borrow_env): - group = Mock() - group._get_backend.return_value = SimpleNamespace(name=lambda: "gloo", comm_ptr=123) - with pytest.raises(RuntimeError, match="Expected a NCCL CUDA backend"): - get_nccl_comm_ptr(group) - borrow_env.assert_not_called() - - -def test_getter_error_propagates(borrow_env): - class Backend: - name = lambda _: "nccl" - _comm_ptr = Mock() - - @property - def comm_ptr(self): - raise RuntimeError("communicator aborted") - - group = Mock() - group._get_backend.return_value = Backend() - with pytest.raises(RuntimeError, match="communicator aborted"): - get_nccl_comm_ptr(group) - Backend._comm_ptr.assert_not_called() - tex.get_nccl_comm_ptr.assert_not_called() - - -def test_cpp_getter_error_propagates(borrow_env): - group = Mock() - backend = SimpleNamespace(name=lambda: "nccl2") - group._get_backend.return_value = backend - tex.get_nccl_comm_ptr.side_effect = RuntimeError("communicator aborted") - with pytest.raises(RuntimeError, match="communicator aborted"): - get_nccl_comm_ptr(group) - tex.get_nccl_comm_ptr.assert_called_once_with(backend) - - -def _run_nccl_rank(rank, world_size, store_path, debug, legacy): - torch.cuda.set_device(rank) - dist.set_debug_level(dist.DebugLevel.DETAIL if debug else dist.DebugLevel.OFF) - backend_name = "nccl-legacy" if legacy and hasattr(dist, "ProcessGroupNCCL2") else "nccl" - dist.init_process_group( - backend_name, - init_method=f"file://{store_path}", - rank=rank, - world_size=world_size, - device_id=torch.device("cuda", rank), - timeout=timedelta(seconds=60), - ) - graph = None - try: - group = dist.group.WORLD - backend = group._get_backend(torch.device("cuda", rank)) - if debug: - assert hasattr(backend, "wrapped_pg") - backend = backend.wrapped_pg - comm = ctypes.c_void_p(get_nccl_comm_ptr(group)) - nccl = ctypes.CDLL("libnccl.so.2") - for name, expected in [ - ("ncclCommCount", world_size), - ("ncclCommUserRank", rank), - ("ncclCommCuDevice", rank), - ]: - query = getattr(nccl, name) - query.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)] - query.restype = ctypes.c_int - value = ctypes.c_int() - assert query(comm, ctypes.byref(value)) == 0 - assert value.value == expected - - nccl.ncclAllReduce.argtypes = [ - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_int, - ctypes.c_int, - ctypes.c_void_p, - ctypes.c_void_p, - ] - nccl.ncclAllReduce.restype = ctypes.c_int - data = torch.full((32,), rank + 1, dtype=torch.float32, device="cuda") - stream = torch.cuda.current_stream().cuda_stream - # NCCL float32 = 7, sum = 0. - assert nccl.ncclAllReduce(data.data_ptr(), data.data_ptr(), 32, 7, 0, comm, stream) == 0 - torch.testing.assert_close(data, torch.full_like(data, world_size * (world_size + 1) / 2)) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - stream = torch.cuda.current_stream().cuda_stream - assert nccl.ncclAllReduce(data.data_ptr(), data.data_ptr(), 32, 7, 0, comm, stream) == 0 - for _ in range(3): - data.fill_(rank + 1) - graph.replay() - torch.testing.assert_close( - data, torch.full_like(data, world_size * (world_size + 1) / 2) - ) - finally: - del graph - dist.destroy_process_group() - - -@pytest.mark.parametrize("world_size", [1, 2]) -@pytest.mark.parametrize("debug", [False, True]) -@pytest.mark.parametrize("legacy", [False, True]) -def test_native_nccl_comm(tmp_path, world_size, debug, legacy): - if not dist.is_nccl_available() or torch.cuda.device_count() < world_size: - pytest.skip(f"Requires NCCL and {world_size} CUDA devices") - if debug and not dist.is_gloo_available(): - pytest.skip("DEBUG DETAIL requires Gloo") - mp.spawn( - _run_nccl_rank, - args=(world_size, str(tmp_path / "store"), debug, legacy), - nprocs=world_size, - )