Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions build_tools/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from importlib import metadata

import setuptools
from packaging.version import Version

from .utils import (
all_files_in_dir,
Expand Down Expand Up @@ -91,10 +92,13 @@ def setup_pytorch_extension(

setup_mpi_flags(include_dirs, cxx_flags)

# ProcessGroupNCCL class declarations are gated on USE_C10D_NCCL. get_nccl_comm_ptr
# borrows the raw ncclComm_t from that backend and is used by both NCCL EP and the
# cuSolverMp path, so the macro is needed regardless of the NCCL EP gate below.
cxx_flags.append("-DUSE_C10D_NCCL")
# Older NVIDIA snapshots also need this fallback despite reporting torch 2.15.
from torch import __version__ as torch_version

if Version(torch_version).release < (2, 16) and not bool(
int(os.getenv("NVTE_TORCH_STABLE_ABI", "0"))
):
cxx_flags.extend(["-DNVTE_WITH_TORCH_NCCL_COMM", "-DUSE_C10D_NCCL"])

# Mirror the NCCL EP gate from setup.py / common CMake. When disabled, the
# ep.cpp source no-ops at the #ifdef boundary; without the define it would
Expand Down
7 changes: 7 additions & 0 deletions docs/installation.rst
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ Additional Prerequisites
1. [For PyTorch support] `PyTorch <https://pytorch.org/>`__ with GPU support.
2. [For JAX support] `JAX <https://github.com/google/jax/>`__ with GPU support, version >= 0.4.7.

PyTorch EP and cuSolverMp use the selected NCCL backend's Python ``comm_ptr``
property or ``_comm_ptr()`` method when available. Builds against PyTorch older
than 2.16 retain a C++ fallback for backends without either Python getter,
including older NVIDIA PyTorch 2.15 snapshots. This fallback is excluded when
building with PyTorch 2.16 or newer, or with ``NVTE_TORCH_STABLE_ABI=1``; those
builds require the backend's Python getter.

Installation (stable release)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Expand Down
7 changes: 4 additions & 3 deletions transformer_engine/pytorch/csrc/extensions.h
Original file line number Diff line number Diff line change
Expand Up @@ -740,10 +740,11 @@ void grouped_swizzle_for_gemm(py::handle &tensor, bool rowwise, bool columnwise)
* Expert Parallelism
**************************************************************************************************/

// Borrow torch's host ncclComm_t from a process group's CUDA backend.
int64_t get_nccl_comm_ptr(c10d::ProcessGroup *process_group);
#ifdef NVTE_WITH_TORCH_NCCL_COMM
int64_t get_nccl_comm_ptr(c10d::Backend *backend);
#endif

// Borrows torch's NCCL host comm (``comm_ptr`` from ``get_nccl_comm_ptr``).
// Borrows torch's NCCL host comm passed from Python.
// ``group_name`` is the PG name used by the symm-mem window resolver.
// ``zero_copy`` is forwarded into ``NVTEEpGroupConfig.zero_copy``.
void ep_initialize(uintptr_t comm_ptr, const std::string &group_name, int64_t num_experts,
Expand Down
14 changes: 8 additions & 6 deletions transformer_engine/pytorch/csrc/extensions/comm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
* See LICENSE for license information.
************************************************************************/

#ifdef NVTE_WITH_TORCH_NCCL_COMM

// NCCL backend headers, for borrowing the raw ncclComm_t. The NCCL2 backend
// header exists only on torch builds that ship it.
#include <torch/csrc/distributed/c10d/ProcessGroupNCCL.hpp>
Expand Down Expand Up @@ -45,18 +47,16 @@ int64_t borrow_comm_ptr(PG* pg) {

} // namespace

int64_t get_nccl_comm_ptr(c10d::ProcessGroup* process_group) {
NVTE_CHECK(process_group != nullptr, "get_nccl_comm_ptr: process_group must be non-null");
auto backend = process_group->getBackend(c10::DeviceType::CUDA);
NVTE_CHECK(backend, "get_nccl_comm_ptr: process group has no CUDA backend");
int64_t get_nccl_comm_ptr(c10d::Backend* backend) {
NVTE_CHECK(backend != nullptr, "get_nccl_comm_ptr: backend must be non-null");

// Classic backend (ProcessGroupNCCL._comm_ptr() in Python).
if (auto* nccl_pg = dynamic_cast<c10d::ProcessGroupNCCL*>(backend.get())) {
if (auto* nccl_pg = dynamic_cast<c10d::ProcessGroupNCCL*>(backend)) {
return borrow_comm_ptr(nccl_pg);
}
// NCCL2 backend, default on recent torch.
#ifdef NVTE_HAS_NCCL2_PG
if (auto* nccl2_pg = dynamic_cast<c10d::nccl2::ProcessGroupNCCL*>(backend.get())) {
if (auto* nccl2_pg = dynamic_cast<c10d::nccl2::ProcessGroupNCCL*>(backend)) {
return borrow_comm_ptr(nccl2_pg);
}
#endif
Expand All @@ -66,3 +66,5 @@ int64_t get_nccl_comm_ptr(c10d::ProcessGroup* process_group) {
}

} // namespace transformer_engine::pytorch

#endif // NVTE_WITH_TORCH_NCCL_COMM
2 changes: 1 addition & 1 deletion transformer_engine/pytorch/csrc/extensions/ep.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ bool ep_zero_copy_supported() {
}

// ── Bootstrap ────────────────────────────────────────────────────────────────
// ``comm_ptr`` is torch's borrowed NCCL host comm (see get_nccl_comm_ptr).
// ``comm_ptr`` is torch's borrowed NCCL host comm passed from Python.
// ``group_name`` is captured for the symm-mem window resolver.

void ep_initialize(uintptr_t comm_ptr, const std::string& group_name, int64_t num_experts,
Expand Down
6 changes: 3 additions & 3 deletions transformer_engine/pytorch/csrc/extensions/pybind.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -726,10 +726,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
&transformer_engine::pytorch::multi_tensor_compute_scale_inv_e8m0_cuda,
"Fused compute E8M0 scale_inv from amax", py::call_guard<py::gil_scoped_release>());

// Borrow torch's host ncclComm_t from a process group (used by NCCL EP and cuSolverMp).
#ifdef NVTE_WITH_TORCH_NCCL_COMM
m.def("get_nccl_comm_ptr", &transformer_engine::pytorch::get_nccl_comm_ptr,
"Borrow torch's host ncclComm_t from a process group's CUDA backend.",
py::arg("process_group"));
"Borrow torch's host ncclComm_t from a CUDA backend.", py::arg("backend"));
#endif

// Newton-Schulz (cuSolverMp)
m.def("cusolvermp_ctx_create", &transformer_engine::pytorch::cusolvermp_ctx_create,
Expand Down
34 changes: 34 additions & 0 deletions transformer_engine/pytorch/distributed.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,40 @@
__all__ = ["checkpoint", "CudaRNGStatesTracker"]


def get_nccl_comm_ptr(group: torch.distributed.ProcessGroup) -> int:
"""Materialize and borrow the current device's NCCL communicator.

This is collective over ``group``. The caller must keep the group alive
until all users of the borrowed communicator have been destroyed.
"""
device = torch.cuda.current_device()
backend = group._get_backend(torch.device("cuda", device))
while hasattr(backend, "wrapped_pg"):
backend = backend.wrapped_pg
if backend.name() not in ("nccl", "nccl2", "nccl-lazy"):
raise RuntimeError(f"Expected a NCCL CUDA backend, got '{backend.name()}'")

# The communicator must exist on this device before either getter is called.
torch.distributed.barrier(group=group, device_ids=[device])
try:
comm_ptr = backend.comm_ptr
except AttributeError:
getter = getattr(backend, "_comm_ptr", None)
if getter is not None:
comm_ptr = getter()
elif hasattr(tex, "get_nccl_comm_ptr"):
comm_ptr = tex.get_nccl_comm_ptr(backend)
else:
raise RuntimeError(
f"PyTorch {torch.__version__} backend '{backend.name()}' does not expose "
"a NCCL communicator pointer. Use a PyTorch build with comm_ptr or "
"_comm_ptr support for this backend."
) from None
if not isinstance(comm_ptr, int) or isinstance(comm_ptr, bool) or comm_ptr <= 0:
raise RuntimeError("NCCL backend returned an invalid communicator pointer")
return comm_ptr


_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {
"tensor_model_parallel": False,
"partition_dim": -1,
Expand Down
6 changes: 2 additions & 4 deletions transformer_engine/pytorch/ep.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from .constants import DType
from .cpu_offload import mark_not_offload
from .distributed import symm_mem_alloc, release_symm_mem_pool
from .distributed import get_nccl_comm_ptr, symm_mem_alloc, release_symm_mem_pool
from .quantized_tensor import QuantizedTensor, QuantizedTensorStorage
from .tensor.storage.mxfp8_tensor_storage import MXFP8TensorStorage

Expand Down Expand Up @@ -198,9 +198,7 @@ def ep_bootstrap(
stacklevel=2,
)

# Materialize the PG's NCCL comm before borrowing its raw handle.
dist.barrier(group=ep_group, device_ids=[torch.cuda.current_device()])
comm_ptr = tex.get_nccl_comm_ptr(ep_group)
comm_ptr = get_nccl_comm_ptr(ep_group)

tex.ep_initialize(
int(comm_ptr),
Expand Down
19 changes: 3 additions & 16 deletions transformer_engine/pytorch/optimizers/newton_schulz.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

import transformer_engine_torch as tex

from ..distributed import get_nccl_comm_ptr


_COEFFICIENT_SETS = {
# Values are rounded to closest representable in single precision.
Expand Down Expand Up @@ -148,7 +150,7 @@ def __init__(self, group: dist.ProcessGroup) -> None:
if self.rank < 0:
raise RuntimeError("The current process is not a member of the supplied process group")

comm_ptr = _get_nccl_comm_ptr(group)
comm_ptr = get_nccl_comm_ptr(group)
self._ptr = tex.cusolvermp_ctx_create(comm_ptr, self.nranks, self.rank)

@property
Expand All @@ -172,21 +174,6 @@ def __del__(self) -> None:
self.destroy()


def _get_nccl_comm_ptr(group: dist.ProcessGroup) -> int:
"""Materialize and borrow a raw NCCL communicator from a process group."""
backend = dist.get_backend(group)
if backend != "nccl":
raise RuntimeError(f"Newton-Schulz requires NCCL backend, got '{backend}'")

# The NCCL backend creates communicators lazily; this collective materializes
# the borrowed communicator on every rank first.
dist.barrier(group=group, device_ids=[torch.cuda.current_device()])
comm_ptr = tex.get_nccl_comm_ptr(group)
if not isinstance(comm_ptr, int) or comm_ptr == 0:
raise RuntimeError("NCCL backend returned an invalid communicator pointer")
return comm_ptr


def newton_schulz(
x: torch.Tensor,
ctx: CusolverMpCtx,
Expand Down