kvm: support Host HA on Ceph RBD primary storage

debian/control

@@ -26,6 +26,7 @@ Package: cloudstack-agent
 Architecture: all
 Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17 | openjdk-21-jre-headless | java21-runtime-headless | java21-runtime | zulu-21, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat
 Recommends: init-system-helpers
+Suggests: python3-rados, python3-rbd
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent
  The CloudStack agent is in charge of managing shared computing resources in

engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java

@@ -33,7 +33,8 @@
  */
 public interface HighAvailabilityManager extends Manager {
 
-    List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint);
+    List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint,
+            StoragePoolType.RBD);
 
     ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false",
         "Force High-Availability to happen even if the VM says no.", true, Cluster);

packaging/el8/cloud.spec

@@ -128,6 +128,8 @@ Requires: (libgcrypt > 1.8.3 or libgcrypt20)
 Requires: (selinux-tools if selinux-tools)
 Requires: sysstat
 Requires: python3-libnbd
+Suggests: python3-rados
+Suggests: python3-rbd
 Requires: socat
 Provides: cloud-agent
 Group: System Environment/Libraries

packaging/suse15/cloud.spec

@@ -120,6 +120,8 @@ Requires: cifs-utils
 Requires: (edk2-ovmf or qemu-ovmf-x86_64)
 Requires: swtpm
 Requires: (python3-libvirt or python3-libvirt-python)
+Suggests: (python3-rados or python3-ceph-common)
+Suggests: (python3-rbd or python3-ceph-common)
 Requires: (qemu-img or qemu-tools)
 Requires: qemu-kvm
 Requires: cryptsetup

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java

@@ -37,23 +37,38 @@ public KVMHAChecker(List<HAStoragePool> pools, HostTO host, boolean reportIfHear
     /*
      * True means heart beating is on going, or we can't get it's status.
      * False means heart beating is stopped definitely.
+     *
+     * The answer comes from the heartbeats only: it is what VM HA uses (KVMInvestigator, CheckOnHostCommand), which needs no out-of-band management.
+     * The check of the activity of the VMs on their volumes (KVMHAVMActivityChecker) is only done by Host HA, which needs it.
+     *
+     * If the host is checked on more than one storage pool, it is dead if its heartbeat is stopped on one of them when
+     * reportIfHeartBeatFailedForOneStoragePool is set, or else only if its heartbeat is stopped on all of them.
      */
     @Override
     public Boolean hasHeartBeat() {
         String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(),
                 storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", ")));
         logger.debug("Checking heart beat with KVMHAChecker for {}", hostAndPools);
 
+        if (storagePools.isEmpty()) {
+            // this host monitors no storage pool, e.g. its agent is not upgraded yet and does not monitor the pools of the type
+            // which a newer version does: it has nothing to tell about the host, which is not a dead host
+            logger.debug("KVMHAChecker has no storage pool to check the heart beat for {}, so it can not tell whether the host is alive.", hostAndPools);
+            return true;
+        }
+
         boolean heartBeatCheckResult = false;
         for (HAStoragePool pool : storagePools) {
-            heartBeatCheckResult = pool.getPool().hasHeartBeat(pool, host);
-            if (reportIfHeartBeatFailedForOneStoragePool && !heartBeatCheckResult) {
+            if (pool.getPool().hasHeartBeat(pool, host)) {
+                heartBeatCheckResult = true;
+            } else if (reportIfHeartBeatFailedForOneStoragePool) {
+                heartBeatCheckResult = false;
                 break;
             }
         }
 
         if (!heartBeatCheckResult) {
-            logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools);
+            logger.warn("All checks with KVMHAChecker for {} considered it as dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostAndPools);
         }
 
         return heartBeatCheckResult;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java

@@ -20,6 +20,7 @@
 import com.cloud.agent.properties.AgentPropertiesFileHandler;
 import com.cloud.ha.HighAvailabilityManager;
 import com.cloud.utils.script.Script;
+import org.apache.commons.lang3.StringUtils;
 import org.libvirt.Connect;
 import org.libvirt.LibvirtException;
 import org.libvirt.StoragePool;
@@ -38,9 +39,15 @@ public class KVMHAMonitor extends KVMHABase implements Runnable {
     private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout;
 
     private final String hostPrivateIp;
+    private final String hostGuid;
 
-    public KVMHAMonitor(String host) {
+    /**
+     * @param host     the private IP address of the host, which names its heartbeat on NFS and SharedMountPoint pools
+     * @param hostGuid the GUID of the host as the management server knows it, which names its heartbeat on RBD pools
+     */
+    public KVMHAMonitor(String host, String hostGuid) {
         hostPrivateIp = host;
+        this.hostGuid = hostGuid;
         rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT);
     }
 
@@ -60,7 +67,11 @@ public void removeStoragePool(String uuid) {
         synchronized (haStoragePools) {
             HAStoragePool pool = haStoragePools.get(uuid);
             if (pool != null) {
-                Script.runSimpleBashScript("umount " + pool.getMountDestPath());
+                if (StringUtils.isNotEmpty(pool.getMountDestPath())) {
+                    Script umount = new Script("umount", logger);
+                    umount.add(pool.getMountDestPath());
+                    umount.execute();
+                }
                 haStoragePools.remove(uuid);
             }
         }
@@ -94,7 +105,11 @@ protected void runHeartBeat() {
 
                 if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) {
                     logger.warn("Write heartbeat for pool [{}] failed: {}; stopping cloudstack-agent.", uuid, result);
-                    primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, null, false);;
+                    try {
+                        primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, null, false);
+                    } catch (RuntimeException e) {
+                        logger.error("Unable to stop the host after the failed heartbeat of pool [{}]", uuid, e);
+                    }
                 }
             }
             if (!removedPools.isEmpty()) {
@@ -107,7 +122,15 @@ protected void runHeartBeat() {
 
     private String executePoolHeartBeatCommand(String uuid, HAStoragePool primaryStoragePool, String result) {
         for (int attempt = 1; attempt <= _heartBeatUpdateMaxTries; attempt++) {
-            result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true);
+            try {
+                String heartBeatHostId = primaryStoragePool.getPool().getHeartBeatHostId(hostPrivateIp, hostGuid);
+                result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, heartBeatHostId, true);
+            } catch (RuntimeException e) {
+                // the heartbeat can not even be started, e.g. the pool is not configured completely or its script is missing:
+                // it is a failure of this pool only, which must not stop the heartbeats of the other pools, and trying again would not help
+                logger.error("Unable to run the heartbeat for pool [{}]: {}", uuid, e.toString(), e);
+                return "unable to run the heartbeat: " + e;
+            }
             if (result == null) {
                 break;
             }
@@ -136,7 +159,7 @@ private void checkForNotExistingLibvirtStoragePools(Set<String> removedPools, St
                 removedPools.add(uuid);
             }
 
-            logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid);
+            logger.debug("Found storage pool [{}] in libvirt, continuing.", uuid);
 
         } catch (LibvirtException e) {
             logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e);
@@ -152,7 +175,12 @@ private void checkForNotExistingLibvirtStoragePools(Set<String> removedPools, St
     public void run() {
         while (true) {
 
-            runHeartBeat();
+            try {
+                runHeartBeat();
+            } catch (RuntimeException e) {
+                // the thread must go on: if it stopped, the heartbeats of all the pools would stop and the host would look dead
+                logger.error("Unexpected error while writing the heartbeats", e);
+            }
 
             try {
                 Thread.sleep(_heartBeatUpdateFreqInMs);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -1491,7 +1491,9 @@ public boolean configure(final String name, final Map<String, Object> params) th
 
         final String[] info = NetUtils.getNetworkParams(privateNic);
 
-        kvmhaMonitor = new KVMHAMonitor(info[0]);
+        // the GUID of the host as the management server knows it, as in Agent.getResourceGuid()
+        final String hostGuid = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.GUID) + "-" + name;
+        kvmhaMonitor = new KVMHAMonitor(info[0], hostGuid);
         final Thread haMonitorThread = new Thread(kvmhaMonitor);
         haMonitorThread.start();
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java

@@ -50,7 +50,15 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L
 
         if (primaryPool.isPoolSupportHA()) {
             final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid());
-            final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds());
+            if (haPool == null) {
+                // e.g. the heartbeat of the pool can not be used on this host, so the pool is not monitored for HA: it is not known whether the host is alive
+                return new Answer(command, false, "CheckVMActivityOnStoragePoolCommand: the storage pool " + pool.getUuid() + " is not monitored for HA on this host");
+            }
+            final String vmActivityCheckPath = primaryPool.getVmActivityCheckPath(libvirtComputingResource.getVmActivityCheckPath());
+            if (vmActivityCheckPath == null) {
+                return new Answer(command, false, "CheckVMActivityOnStoragePoolCommand: unable to find the script which checks the VM activity on a storage pool of type " + pool.getType());
+            }
+            final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds());
             final Future<Boolean> future = executors.submit(ha);
             try {
                 final Boolean result = future.get();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java

@@ -155,6 +155,32 @@ default String getLocalPathFor(String relativePath) {
 
     String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation);
 
+    /**
+     * The id which names the heartbeat of a host on this storage pool: by default the private IP address of the host.
+     *
+     * @param hostPrivateIp the private IP address of the host
+     * @param hostGuid      the GUID of the host as the management server knows it
+     */
+    default String getHeartBeatHostId(String hostPrivateIp, String hostGuid) {
+        return hostPrivateIp;
+    }
+
+    /**
+     * The script which checks the VM activity on this storage pool: by default the one given, which is the script of the
+     * pools with a shared mount point. Null if it can not be found.
+     */
+    default String getVmActivityCheckPath(String defaultVmActivityCheckPath) {
+        return defaultVmActivityCheckPath;
+    }
+
+    /**
+     * Returns why the heartbeat of this storage pool can not be used on this host, or null if it can. A host which can not
+     * write the heartbeat would be fenced, so such a pool is not monitored for HA.
+     */
+    default String getHeartBeatPrerequisiteError() {
+        return null;
+    }
+
     String getStorageNodeId();
 
     Boolean hasHeartBeat(HAStoragePool pool, HostTO host);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java

@@ -37,6 +37,7 @@
 
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.ha.HighAvailabilityManager;
 import com.cloud.hypervisor.kvm.resource.KVMHABase;
 import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType;
 import com.cloud.hypervisor.kvm.resource.KVMHAMonitor;
@@ -428,8 +429,14 @@ private synchronized KVMStoragePool createStoragePool(String name, String host,
 
         // LibvirtStorageAdaptor-specific statement
         if (pool.isPoolSupportHA() && primaryStorage) {
-            KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, PoolType.PrimaryStorage);
-            _haMonitor.addStoragePool(storagePool);
+            String heartBeatPrerequisiteError = pool.getHeartBeatPrerequisiteError();
+            if (heartBeatPrerequisiteError == null) {
+                KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, PoolType.PrimaryStorage);
+                _haMonitor.addStoragePool(storagePool);
+            } else {
+                // without the heartbeat the host would fail to write it at each cycle, and would be fenced
+                logger.error("The storage pool {} is not monitored for HA on this host, as its heartbeat can not be used: {}", pool.getUuid(), heartBeatPrerequisiteError);
+            }
         }
         StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, primaryStorage);
         addStoragePool(pool.getUuid(), info);
@@ -445,7 +452,7 @@ public boolean disconnectPhysicalDisk(StoragePoolType type, String poolUuid, Str
 
     public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) {
         StorageAdaptor adaptor = getStorageAdaptor(type);
-        if (type == StoragePoolType.NetworkFilesystem) {
+        if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) {
             _haMonitor.unregisterStoragePool(uuid);
         }
         boolean deleteStatus = adaptor.deleteStoragePool(uuid);
@@ -458,7 +465,7 @@ public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid)
     public boolean deleteStoragePool(StoragePoolType type, String uuid, Map<String, String> details) {
         StorageAdaptor adaptor = getStorageAdaptor(type);
         boolean deleteStatus;
-        if (type == StoragePoolType.NetworkFilesystem) {
+        if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) {
             _haMonitor.unregisterStoragePool(uuid);
             deleteStatus = adaptor.deleteStoragePool(uuid);
         } else {