diff --git a/build_tools/pytorch.py b/build_tools/pytorch.py
index 173f9cee708..245e922a9c9 100644
--- a/build_tools/pytorch.py
+++ b/build_tools/pytorch.py
@@ -9,6 +9,7 @@
from importlib import metadata
import setuptools
+from packaging.version import Version
from .utils import (
all_files_in_dir,
@@ -91,10 +92,13 @@ def setup_pytorch_extension(
setup_mpi_flags(include_dirs, cxx_flags)
- # ProcessGroupNCCL class declarations are gated on USE_C10D_NCCL. get_nccl_comm_ptr
- # borrows the raw ncclComm_t from that backend and is used by both NCCL EP and the
- # cuSolverMp path, so the macro is needed regardless of the NCCL EP gate below.
- cxx_flags.append("-DUSE_C10D_NCCL")
+ # Older NVIDIA snapshots also need this fallback despite reporting torch 2.15.
+ from torch import __version__ as torch_version
+
+ if Version(torch_version).release < (2, 16) and not bool(
+ int(os.getenv("NVTE_TORCH_STABLE_ABI", "0"))
+ ):
+ cxx_flags.extend(["-DNVTE_WITH_TORCH_NCCL_COMM", "-DUSE_C10D_NCCL"])
# Mirror the NCCL EP gate from setup.py / common CMake. When disabled, the
# ep.cpp source no-ops at the #ifdef boundary; without the define it would
diff --git a/docs/installation.rst b/docs/installation.rst
index 9d3eada827c..513a49b888d 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -55,6 +55,13 @@ Additional Prerequisites
1. [For PyTorch support] `PyTorch `__ with GPU support.
2. [For JAX support] `JAX `__ with GPU support, version >= 0.4.7.
+PyTorch EP and cuSolverMp use the selected NCCL backend's Python ``comm_ptr``
+property or ``_comm_ptr()`` method when available. Builds against PyTorch older
+than 2.16 retain a C++ fallback for backends without either Python getter,
+including older NVIDIA PyTorch 2.15 snapshots. This fallback is excluded when
+building with PyTorch 2.16 or newer, or with ``NVTE_TORCH_STABLE_ABI=1``; those
+builds require the backend's Python getter.
+
Installation (stable release)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h
index d3e7763a7b5..ce92063f3f8 100644
--- a/transformer_engine/pytorch/csrc/extensions.h
+++ b/transformer_engine/pytorch/csrc/extensions.h
@@ -740,10 +740,11 @@ void grouped_swizzle_for_gemm(py::handle &tensor, bool rowwise, bool columnwise)
* Expert Parallelism
**************************************************************************************************/
-// Borrow torch's host ncclComm_t from a process group's CUDA backend.
-int64_t get_nccl_comm_ptr(c10d::ProcessGroup *process_group);
+#ifdef NVTE_WITH_TORCH_NCCL_COMM
+int64_t get_nccl_comm_ptr(c10d::Backend *backend);
+#endif
-// Borrows torch's NCCL host comm (``comm_ptr`` from ``get_nccl_comm_ptr``).
+// Borrows torch's NCCL host comm passed from Python.
// ``group_name`` is the PG name used by the symm-mem window resolver.
// ``zero_copy`` is forwarded into ``NVTEEpGroupConfig.zero_copy``.
void ep_initialize(uintptr_t comm_ptr, const std::string &group_name, int64_t num_experts,
diff --git a/transformer_engine/pytorch/csrc/extensions/comm.cpp b/transformer_engine/pytorch/csrc/extensions/comm.cpp
index ebdddb2dd7e..206bf99fdf1 100644
--- a/transformer_engine/pytorch/csrc/extensions/comm.cpp
+++ b/transformer_engine/pytorch/csrc/extensions/comm.cpp
@@ -4,6 +4,8 @@
* See LICENSE for license information.
************************************************************************/
+#ifdef NVTE_WITH_TORCH_NCCL_COMM
+
// NCCL backend headers, for borrowing the raw ncclComm_t. The NCCL2 backend
// header exists only on torch builds that ship it.
#include
@@ -45,18 +47,16 @@ int64_t borrow_comm_ptr(PG* pg) {
} // namespace
-int64_t get_nccl_comm_ptr(c10d::ProcessGroup* process_group) {
- NVTE_CHECK(process_group != nullptr, "get_nccl_comm_ptr: process_group must be non-null");
- auto backend = process_group->getBackend(c10::DeviceType::CUDA);
- NVTE_CHECK(backend, "get_nccl_comm_ptr: process group has no CUDA backend");
+int64_t get_nccl_comm_ptr(c10d::Backend* backend) {
+ NVTE_CHECK(backend != nullptr, "get_nccl_comm_ptr: backend must be non-null");
// Classic backend (ProcessGroupNCCL._comm_ptr() in Python).
- if (auto* nccl_pg = dynamic_cast(backend.get())) {
+ if (auto* nccl_pg = dynamic_cast(backend)) {
return borrow_comm_ptr(nccl_pg);
}
// NCCL2 backend, default on recent torch.
#ifdef NVTE_HAS_NCCL2_PG
- if (auto* nccl2_pg = dynamic_cast(backend.get())) {
+ if (auto* nccl2_pg = dynamic_cast(backend)) {
return borrow_comm_ptr(nccl2_pg);
}
#endif
@@ -66,3 +66,5 @@ int64_t get_nccl_comm_ptr(c10d::ProcessGroup* process_group) {
}
} // namespace transformer_engine::pytorch
+
+#endif // NVTE_WITH_TORCH_NCCL_COMM
diff --git a/transformer_engine/pytorch/csrc/extensions/ep.cpp b/transformer_engine/pytorch/csrc/extensions/ep.cpp
index 66316a07ccd..65391c1121b 100644
--- a/transformer_engine/pytorch/csrc/extensions/ep.cpp
+++ b/transformer_engine/pytorch/csrc/extensions/ep.cpp
@@ -180,7 +180,7 @@ bool ep_zero_copy_supported() {
}
// ── Bootstrap ────────────────────────────────────────────────────────────────
-// ``comm_ptr`` is torch's borrowed NCCL host comm (see get_nccl_comm_ptr).
+// ``comm_ptr`` is torch's borrowed NCCL host comm passed from Python.
// ``group_name`` is captured for the symm-mem window resolver.
void ep_initialize(uintptr_t comm_ptr, const std::string& group_name, int64_t num_experts,
diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp
index 89e7d92aca9..801bf1d8db0 100644
--- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp
+++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp
@@ -726,10 +726,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
&transformer_engine::pytorch::multi_tensor_compute_scale_inv_e8m0_cuda,
"Fused compute E8M0 scale_inv from amax", py::call_guard());
- // Borrow torch's host ncclComm_t from a process group (used by NCCL EP and cuSolverMp).
+#ifdef NVTE_WITH_TORCH_NCCL_COMM
m.def("get_nccl_comm_ptr", &transformer_engine::pytorch::get_nccl_comm_ptr,
- "Borrow torch's host ncclComm_t from a process group's CUDA backend.",
- py::arg("process_group"));
+ "Borrow torch's host ncclComm_t from a CUDA backend.", py::arg("backend"));
+#endif
// Newton-Schulz (cuSolverMp)
m.def("cusolvermp_ctx_create", &transformer_engine::pytorch::cusolvermp_ctx_create,
diff --git a/transformer_engine/pytorch/distributed.py b/transformer_engine/pytorch/distributed.py
index e90b17e4d9f..41a9e2fc49e 100644
--- a/transformer_engine/pytorch/distributed.py
+++ b/transformer_engine/pytorch/distributed.py
@@ -54,6 +54,40 @@
__all__ = ["checkpoint", "CudaRNGStatesTracker"]
+def get_nccl_comm_ptr(group: torch.distributed.ProcessGroup) -> int:
+ """Materialize and borrow the current device's NCCL communicator.
+
+ This is collective over ``group``. The caller must keep the group alive
+ until all users of the borrowed communicator have been destroyed.
+ """
+ device = torch.cuda.current_device()
+ backend = group._get_backend(torch.device("cuda", device))
+ while hasattr(backend, "wrapped_pg"):
+ backend = backend.wrapped_pg
+ if backend.name() not in ("nccl", "nccl2", "nccl-lazy"):
+ raise RuntimeError(f"Expected a NCCL CUDA backend, got '{backend.name()}'")
+
+ # The communicator must exist on this device before either getter is called.
+ torch.distributed.barrier(group=group, device_ids=[device])
+ try:
+ comm_ptr = backend.comm_ptr
+ except AttributeError:
+ getter = getattr(backend, "_comm_ptr", None)
+ if getter is not None:
+ comm_ptr = getter()
+ elif hasattr(tex, "get_nccl_comm_ptr"):
+ comm_ptr = tex.get_nccl_comm_ptr(backend)
+ else:
+ raise RuntimeError(
+ f"PyTorch {torch.__version__} backend '{backend.name()}' does not expose "
+ "a NCCL communicator pointer. Use a PyTorch build with comm_ptr or "
+ "_comm_ptr support for this backend."
+ ) from None
+ if not isinstance(comm_ptr, int) or isinstance(comm_ptr, bool) or comm_ptr <= 0:
+ raise RuntimeError("NCCL backend returned an invalid communicator pointer")
+ return comm_ptr
+
+
_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {
"tensor_model_parallel": False,
"partition_dim": -1,
diff --git a/transformer_engine/pytorch/ep.py b/transformer_engine/pytorch/ep.py
index 9ed10fb4721..a7209fd8d84 100644
--- a/transformer_engine/pytorch/ep.py
+++ b/transformer_engine/pytorch/ep.py
@@ -17,7 +17,7 @@
from .constants import DType
from .cpu_offload import mark_not_offload
-from .distributed import symm_mem_alloc, release_symm_mem_pool
+from .distributed import get_nccl_comm_ptr, symm_mem_alloc, release_symm_mem_pool
from .quantized_tensor import QuantizedTensor, QuantizedTensorStorage
from .tensor.storage.mxfp8_tensor_storage import MXFP8TensorStorage
@@ -198,9 +198,7 @@ def ep_bootstrap(
stacklevel=2,
)
- # Materialize the PG's NCCL comm before borrowing its raw handle.
- dist.barrier(group=ep_group, device_ids=[torch.cuda.current_device()])
- comm_ptr = tex.get_nccl_comm_ptr(ep_group)
+ comm_ptr = get_nccl_comm_ptr(ep_group)
tex.ep_initialize(
int(comm_ptr),
diff --git a/transformer_engine/pytorch/optimizers/newton_schulz.py b/transformer_engine/pytorch/optimizers/newton_schulz.py
index e1b1a0bcbde..a7843e580a4 100644
--- a/transformer_engine/pytorch/optimizers/newton_schulz.py
+++ b/transformer_engine/pytorch/optimizers/newton_schulz.py
@@ -12,6 +12,8 @@
import transformer_engine_torch as tex
+from ..distributed import get_nccl_comm_ptr
+
_COEFFICIENT_SETS = {
# Values are rounded to closest representable in single precision.
@@ -148,7 +150,7 @@ def __init__(self, group: dist.ProcessGroup) -> None:
if self.rank < 0:
raise RuntimeError("The current process is not a member of the supplied process group")
- comm_ptr = _get_nccl_comm_ptr(group)
+ comm_ptr = get_nccl_comm_ptr(group)
self._ptr = tex.cusolvermp_ctx_create(comm_ptr, self.nranks, self.rank)
@property
@@ -172,21 +174,6 @@ def __del__(self) -> None:
self.destroy()
-def _get_nccl_comm_ptr(group: dist.ProcessGroup) -> int:
- """Materialize and borrow a raw NCCL communicator from a process group."""
- backend = dist.get_backend(group)
- if backend != "nccl":
- raise RuntimeError(f"Newton-Schulz requires NCCL backend, got '{backend}'")
-
- # The NCCL backend creates communicators lazily; this collective materializes
- # the borrowed communicator on every rank first.
- dist.barrier(group=group, device_ids=[torch.cuda.current_device()])
- comm_ptr = tex.get_nccl_comm_ptr(group)
- if not isinstance(comm_ptr, int) or comm_ptr == 0:
- raise RuntimeError("NCCL backend returned an invalid communicator pointer")
- return comm_ptr
-
-
def newton_schulz(
x: torch.Tensor,
ctx: CusolverMpCtx,