diff --git a/spark-rapids/README.md b/spark-rapids/README.md index 92f55fea3..c55dc59ed 100644 --- a/spark-rapids/README.md +++ b/spark-rapids/README.md @@ -106,6 +106,10 @@ profile id or the name (ie 3g.20gb) can be specified. For example: --metadata=^:^MIG_CGI='3g.20gb,9' ``` Create cluster with MIG enabled: + + The MIG startup script runs only on worker nodes. It skips the master node + so that the required MIG reboot does not interrupt Dataproc master service + initialization. ```bash REGION= diff --git a/spark-rapids/mig.sh b/spark-rapids/mig.sh index 6d0874ff3..1aa490e9a 100644 --- a/spark-rapids/mig.sh +++ b/spark-rapids/mig.sh @@ -26,6 +26,8 @@ set -euxo pipefail +readonly ROLE="$(/usr/share/google/get_metadata_value attributes/dataproc-role)" + function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs } @@ -172,11 +174,11 @@ function configure_dkms_certs() { function clear_dkms_key { if [[ -z "${PSN}" ]]; then - echo "No signing secret provided. skipping" >2 + echo "No signing secret provided. skipping" >&2 return 0 fi - echo "WARN -- PURGING SIGNING MATERIAL -- WARN" >2 - echo "future dkms runs will not use correct signing key" >2 + echo "WARN -- PURGING SIGNING MATERIAL -- WARN" >&2 + echo "future dkms runs will not use correct signing key" >&2 rm -rf "${CA_TMPDIR}" /var/lib/dkms/mok.key /var/lib/shim-signed/mok/MOK.priv } @@ -217,6 +219,17 @@ NVIDIA_DRIVER_VERSION=$(get_metadata_attribute 'driver-version' '550.54.15') #53 CUDA_VERSION_MAJOR="${CUDA_VERSION%.*}" #12.2 # EXCEPTIONS +# Debian 12 security kernels use the four-argument pci_resize_resource API, +# which is incompatible with NVIDIA 550 open modules. +if is_debian12 ; then + NVIDIA_DRIVER_VERSION=$(get_metadata_attribute 'driver-version' '580.95.05') + if [[ "${NVIDIA_DRIVER_VERSION%%.*}" == "550" ]]; then + echo "WARNING: Driver version 550 is incompatible with Debian 12 kernel. Overriding to 580.95.05" >&2 + NVIDIA_DRIVER_VERSION='580.95.05' + fi + USE_REPO_INSTALL="true" +fi + # Change CUDA version for Ubuntu 18 (Cuda 12.1.1 - Driver v530.30.02 is the latest version supported by Ubuntu 18) if [[ "${OS_NAME}" == "ubuntu" ]]; then if is_ubuntu18 ; then @@ -296,30 +309,51 @@ function install_nvidia_gpu_driver() { execute_with_retries "apt-get install -y -q 'linux-headers-$(uname -r)'" - curl -fsSL --retry-connrefused --retry 3 --retry-max-time 5 \ - "${LOCAL_DEB_URL}" -o /tmp/local-installer.deb + if [[ "${USE_REPO_INSTALL:-false}" == "true" ]]; then + execute_with_retries \ + "curl -fsSL --retry-connrefused --retry 3 --retry-max-time 5 https://developer.download.nvidia.com/compute/cuda/repos/${shortname}/x86_64/cuda-keyring_1.1-1_all.deb -o /tmp/cuda-keyring_1.1-1_all.deb" + execute_with_retries "dpkg -i /tmp/cuda-keyring_1.1-1_all.deb" + rm -f /tmp/cuda-keyring_1.1-1_all.deb + add_contrib_components + execute_with_retries "apt-get update" - dpkg -i /tmp/local-installer.deb - rm /tmp/local-installer.deb - cp ${DIST_KEYRING_DIR}/cuda-*-keyring.gpg /usr/share/keyrings/ + execute_with_retries "apt-get install -y -q --no-install-recommends dkms" + execute_with_retries \ + "apt-get install -y -q --no-install-recommends nvidia-driver-pinning-${NVIDIA_DRIVER_VERSION_PREFIX}" - add_contrib_components + configure_dkms_certs + execute_with_retries \ + "apt-get install -y -q --no-install-recommends nvidia-kernel-open-dkms nvidia-driver-cuda" + clear_dkms_key - execute_with_retries "apt-get update" + execute_with_retries \ + "apt-get install -y -q --no-install-recommends cuda-toolkit-${CUDA_VERSION_MAJOR//./-}" + else + curl -fsSL --retry-connrefused --retry 3 --retry-max-time 5 \ + "${LOCAL_DEB_URL}" -o /tmp/local-installer.deb - ## EXCEPTION - if is_debian10 ; then - apt-get remove -y libglvnd0 - apt-get install -y ca-certificates-java - fi + dpkg -i /tmp/local-installer.deb + rm /tmp/local-installer.deb + cp ${DIST_KEYRING_DIR}/cuda-*-keyring.gpg /usr/share/keyrings/ - configure_dkms_certs - execute_with_retries "apt-get install -y -q nvidia-kernel-open-dkms" - clear_dkms_key - execute_with_retries \ - "apt-get install -y -q --no-install-recommends cuda-drivers-${NVIDIA_DRIVER_VERSION_PREFIX}" - execute_with_retries \ - "apt-get install -y -q --no-install-recommends cuda-toolkit-${CUDA_VERSION_MAJOR//./-}" + add_contrib_components + + execute_with_retries "apt-get update" + + ## EXCEPTION + if is_debian10 ; then + apt-get remove -y libglvnd0 + apt-get install -y ca-certificates-java + fi + + configure_dkms_certs + execute_with_retries "apt-get install -y -q nvidia-kernel-open-dkms" + clear_dkms_key + execute_with_retries \ + "apt-get install -y -q --no-install-recommends cuda-drivers-${NVIDIA_DRIVER_VERSION_PREFIX}" + execute_with_retries \ + "apt-get install -y -q --no-install-recommends cuda-toolkit-${CUDA_VERSION_MAJOR//./-}" + fi modprobe nvidia @@ -503,6 +537,13 @@ function remove_old_backports { } function main() { + # Rebooting a master before Dataproc initialization completes causes its + # master-only services to be initialized twice. MIG is only needed on workers. + if [[ "${ROLE}" == "Master" ]]; then + echo "Skipping MIG setup and reboot on Dataproc master" + return + fi + if [[ ${OS_NAME} == debian ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} <= 2.1" | bc -l) == 1 ]]; then remove_old_backports fi diff --git a/spark-rapids/spark-rapids.sh b/spark-rapids/spark-rapids.sh index 760f8259e..9df7bb8b2 100644 --- a/spark-rapids/spark-rapids.sh +++ b/spark-rapids/spark-rapids.sh @@ -367,7 +367,7 @@ else fi # Update SPARK RAPIDS config -readonly HARDCODED_RAPIDS_VERSION="26.08.1" +readonly HARDCODED_RAPIDS_VERSION="26.10.0" # 1. Try to get explicit version from GCE Metadata SPARK_RAPIDS_VERSION=$(get_metadata_attribute 'spark-rapids-version' '')