diff --git a/build_tools/pytorch.py b/build_tools/pytorch.py index 173f9cee708..245e922a9c9 100644 --- a/build_tools/pytorch.py +++ b/build_tools/pytorch.py @@ -9,6 +9,7 @@ from importlib import metadata import setuptools +from packaging.version import Version from .utils import ( all_files_in_dir, @@ -91,10 +92,13 @@ def setup_pytorch_extension( setup_mpi_flags(include_dirs, cxx_flags) - # ProcessGroupNCCL class declarations are gated on USE_C10D_NCCL. get_nccl_comm_ptr - # borrows the raw ncclComm_t from that backend and is used by both NCCL EP and the - # cuSolverMp path, so the macro is needed regardless of the NCCL EP gate below. - cxx_flags.append("-DUSE_C10D_NCCL") + # Older NVIDIA snapshots also need this fallback despite reporting torch 2.15. + from torch import __version__ as torch_version + + if Version(torch_version).release < (2, 16) and not bool( + int(os.getenv("NVTE_TORCH_STABLE_ABI", "0")) + ): + cxx_flags.extend(["-DNVTE_WITH_TORCH_NCCL_COMM", "-DUSE_C10D_NCCL"]) # Mirror the NCCL EP gate from setup.py / common CMake. When disabled, the # ep.cpp source no-ops at the #ifdef boundary; without the define it would diff --git a/docs/installation.rst b/docs/installation.rst index 9d3eada827c..513a49b888d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -55,6 +55,13 @@ Additional Prerequisites 1. [For PyTorch support] `PyTorch `__ with GPU support. 2. [For JAX support] `JAX `__ with GPU support, version >= 0.4.7. +PyTorch EP and cuSolverMp use the selected NCCL backend's Python ``comm_ptr`` +property or ``_comm_ptr()`` method when available. Builds against PyTorch older +than 2.16 retain a C++ fallback for backends without either Python getter, +including older NVIDIA PyTorch 2.15 snapshots. This fallback is excluded when +building with PyTorch 2.16 or newer, or with ``NVTE_TORCH_STABLE_ABI=1``; those +builds require the backend's Python getter. + Installation (stable release) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index d3e7763a7b5..ce92063f3f8 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -740,10 +740,11 @@ void grouped_swizzle_for_gemm(py::handle &tensor, bool rowwise, bool columnwise) * Expert Parallelism **************************************************************************************************/ -// Borrow torch's host ncclComm_t from a process group's CUDA backend. -int64_t get_nccl_comm_ptr(c10d::ProcessGroup *process_group); +#ifdef NVTE_WITH_TORCH_NCCL_COMM +int64_t get_nccl_comm_ptr(c10d::Backend *backend); +#endif -// Borrows torch's NCCL host comm (``comm_ptr`` from ``get_nccl_comm_ptr``). +// Borrows torch's NCCL host comm passed from Python. // ``group_name`` is the PG name used by the symm-mem window resolver. // ``zero_copy`` is forwarded into ``NVTEEpGroupConfig.zero_copy``. void ep_initialize(uintptr_t comm_ptr, const std::string &group_name, int64_t num_experts, diff --git a/transformer_engine/pytorch/csrc/extensions/comm.cpp b/transformer_engine/pytorch/csrc/extensions/comm.cpp index ebdddb2dd7e..206bf99fdf1 100644 --- a/transformer_engine/pytorch/csrc/extensions/comm.cpp +++ b/transformer_engine/pytorch/csrc/extensions/comm.cpp @@ -4,6 +4,8 @@ * See LICENSE for license information. ************************************************************************/ +#ifdef NVTE_WITH_TORCH_NCCL_COMM + // NCCL backend headers, for borrowing the raw ncclComm_t. The NCCL2 backend // header exists only on torch builds that ship it. #include @@ -45,18 +47,16 @@ int64_t borrow_comm_ptr(PG* pg) { } // namespace -int64_t get_nccl_comm_ptr(c10d::ProcessGroup* process_group) { - NVTE_CHECK(process_group != nullptr, "get_nccl_comm_ptr: process_group must be non-null"); - auto backend = process_group->getBackend(c10::DeviceType::CUDA); - NVTE_CHECK(backend, "get_nccl_comm_ptr: process group has no CUDA backend"); +int64_t get_nccl_comm_ptr(c10d::Backend* backend) { + NVTE_CHECK(backend != nullptr, "get_nccl_comm_ptr: backend must be non-null"); // Classic backend (ProcessGroupNCCL._comm_ptr() in Python). - if (auto* nccl_pg = dynamic_cast(backend.get())) { + if (auto* nccl_pg = dynamic_cast(backend)) { return borrow_comm_ptr(nccl_pg); } // NCCL2 backend, default on recent torch. #ifdef NVTE_HAS_NCCL2_PG - if (auto* nccl2_pg = dynamic_cast(backend.get())) { + if (auto* nccl2_pg = dynamic_cast(backend)) { return borrow_comm_ptr(nccl2_pg); } #endif @@ -66,3 +66,5 @@ int64_t get_nccl_comm_ptr(c10d::ProcessGroup* process_group) { } } // namespace transformer_engine::pytorch + +#endif // NVTE_WITH_TORCH_NCCL_COMM diff --git a/transformer_engine/pytorch/csrc/extensions/ep.cpp b/transformer_engine/pytorch/csrc/extensions/ep.cpp index 66316a07ccd..65391c1121b 100644 --- a/transformer_engine/pytorch/csrc/extensions/ep.cpp +++ b/transformer_engine/pytorch/csrc/extensions/ep.cpp @@ -180,7 +180,7 @@ bool ep_zero_copy_supported() { } // ── Bootstrap ──────────────────────────────────────────────────────────────── -// ``comm_ptr`` is torch's borrowed NCCL host comm (see get_nccl_comm_ptr). +// ``comm_ptr`` is torch's borrowed NCCL host comm passed from Python. // ``group_name`` is captured for the symm-mem window resolver. void ep_initialize(uintptr_t comm_ptr, const std::string& group_name, int64_t num_experts, diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 89e7d92aca9..801bf1d8db0 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -726,10 +726,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { &transformer_engine::pytorch::multi_tensor_compute_scale_inv_e8m0_cuda, "Fused compute E8M0 scale_inv from amax", py::call_guard()); - // Borrow torch's host ncclComm_t from a process group (used by NCCL EP and cuSolverMp). +#ifdef NVTE_WITH_TORCH_NCCL_COMM m.def("get_nccl_comm_ptr", &transformer_engine::pytorch::get_nccl_comm_ptr, - "Borrow torch's host ncclComm_t from a process group's CUDA backend.", - py::arg("process_group")); + "Borrow torch's host ncclComm_t from a CUDA backend.", py::arg("backend")); +#endif // Newton-Schulz (cuSolverMp) m.def("cusolvermp_ctx_create", &transformer_engine::pytorch::cusolvermp_ctx_create, diff --git a/transformer_engine/pytorch/distributed.py b/transformer_engine/pytorch/distributed.py index e90b17e4d9f..41a9e2fc49e 100644 --- a/transformer_engine/pytorch/distributed.py +++ b/transformer_engine/pytorch/distributed.py @@ -54,6 +54,40 @@ __all__ = ["checkpoint", "CudaRNGStatesTracker"] +def get_nccl_comm_ptr(group: torch.distributed.ProcessGroup) -> int: + """Materialize and borrow the current device's NCCL communicator. + + This is collective over ``group``. The caller must keep the group alive + until all users of the borrowed communicator have been destroyed. + """ + device = torch.cuda.current_device() + backend = group._get_backend(torch.device("cuda", device)) + while hasattr(backend, "wrapped_pg"): + backend = backend.wrapped_pg + if backend.name() not in ("nccl", "nccl2", "nccl-lazy"): + raise RuntimeError(f"Expected a NCCL CUDA backend, got '{backend.name()}'") + + # The communicator must exist on this device before either getter is called. + torch.distributed.barrier(group=group, device_ids=[device]) + try: + comm_ptr = backend.comm_ptr + except AttributeError: + getter = getattr(backend, "_comm_ptr", None) + if getter is not None: + comm_ptr = getter() + elif hasattr(tex, "get_nccl_comm_ptr"): + comm_ptr = tex.get_nccl_comm_ptr(backend) + else: + raise RuntimeError( + f"PyTorch {torch.__version__} backend '{backend.name()}' does not expose " + "a NCCL communicator pointer. Use a PyTorch build with comm_ptr or " + "_comm_ptr support for this backend." + ) from None + if not isinstance(comm_ptr, int) or isinstance(comm_ptr, bool) or comm_ptr <= 0: + raise RuntimeError("NCCL backend returned an invalid communicator pointer") + return comm_ptr + + _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = { "tensor_model_parallel": False, "partition_dim": -1, diff --git a/transformer_engine/pytorch/ep.py b/transformer_engine/pytorch/ep.py index 9ed10fb4721..a7209fd8d84 100644 --- a/transformer_engine/pytorch/ep.py +++ b/transformer_engine/pytorch/ep.py @@ -17,7 +17,7 @@ from .constants import DType from .cpu_offload import mark_not_offload -from .distributed import symm_mem_alloc, release_symm_mem_pool +from .distributed import get_nccl_comm_ptr, symm_mem_alloc, release_symm_mem_pool from .quantized_tensor import QuantizedTensor, QuantizedTensorStorage from .tensor.storage.mxfp8_tensor_storage import MXFP8TensorStorage @@ -198,9 +198,7 @@ def ep_bootstrap( stacklevel=2, ) - # Materialize the PG's NCCL comm before borrowing its raw handle. - dist.barrier(group=ep_group, device_ids=[torch.cuda.current_device()]) - comm_ptr = tex.get_nccl_comm_ptr(ep_group) + comm_ptr = get_nccl_comm_ptr(ep_group) tex.ep_initialize( int(comm_ptr), diff --git a/transformer_engine/pytorch/optimizers/newton_schulz.py b/transformer_engine/pytorch/optimizers/newton_schulz.py index e1b1a0bcbde..a7843e580a4 100644 --- a/transformer_engine/pytorch/optimizers/newton_schulz.py +++ b/transformer_engine/pytorch/optimizers/newton_schulz.py @@ -12,6 +12,8 @@ import transformer_engine_torch as tex +from ..distributed import get_nccl_comm_ptr + _COEFFICIENT_SETS = { # Values are rounded to closest representable in single precision. @@ -148,7 +150,7 @@ def __init__(self, group: dist.ProcessGroup) -> None: if self.rank < 0: raise RuntimeError("The current process is not a member of the supplied process group") - comm_ptr = _get_nccl_comm_ptr(group) + comm_ptr = get_nccl_comm_ptr(group) self._ptr = tex.cusolvermp_ctx_create(comm_ptr, self.nranks, self.rank) @property @@ -172,21 +174,6 @@ def __del__(self) -> None: self.destroy() -def _get_nccl_comm_ptr(group: dist.ProcessGroup) -> int: - """Materialize and borrow a raw NCCL communicator from a process group.""" - backend = dist.get_backend(group) - if backend != "nccl": - raise RuntimeError(f"Newton-Schulz requires NCCL backend, got '{backend}'") - - # The NCCL backend creates communicators lazily; this collective materializes - # the borrowed communicator on every rank first. - dist.barrier(group=group, device_ids=[torch.cuda.current_device()]) - comm_ptr = tex.get_nccl_comm_ptr(group) - if not isinstance(comm_ptr, int) or comm_ptr == 0: - raise RuntimeError("NCCL backend returned an invalid communicator pointer") - return comm_ptr - - def newton_schulz( x: torch.Tensor, ctx: CusolverMpCtx,