diff --git a/debian/control b/debian/control index 30504d4c981b..3e9a6a358d29 100644 --- a/debian/control +++ b/debian/control @@ -26,6 +26,7 @@ Package: cloudstack-agent Architecture: all Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17 | openjdk-21-jre-headless | java21-runtime-headless | java21-runtime | zulu-21, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat Recommends: init-system-helpers +Suggests: python3-rados, python3-rbd Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent The CloudStack agent is in charge of managing shared computing resources in diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index 53bfcce27038..f4f8d342eda7 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -33,7 +33,8 @@ */ public interface HighAvailabilityManager extends Manager { - List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint); + List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint, + StoragePoolType.RBD); ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 9d8cab846164..7be4172f7666 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -128,6 +128,8 @@ Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat Requires: python3-libnbd +Suggests: python3-rados +Suggests: python3-rbd Requires: socat Provides: cloud-agent Group: System Environment/Libraries diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec index 04ab66904287..6d6039051e98 100644 --- a/packaging/suse15/cloud.spec +++ b/packaging/suse15/cloud.spec @@ -120,6 +120,8 @@ Requires: cifs-utils Requires: (edk2-ovmf or qemu-ovmf-x86_64) Requires: swtpm Requires: (python3-libvirt or python3-libvirt-python) +Suggests: (python3-rados or python3-ceph-common) +Suggests: (python3-rbd or python3-ceph-common) Requires: (qemu-img or qemu-tools) Requires: qemu-kvm Requires: cryptsetup diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 0ee59c95da39..41beed3bc79e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -37,6 +37,12 @@ public KVMHAChecker(List pools, HostTO host, boolean reportIfHear /* * True means heart beating is on going, or we can't get it's status. * False means heart beating is stopped definitely. + * + * The answer comes from the heartbeats only: it is what VM HA uses (KVMInvestigator, CheckOnHostCommand), which needs no out-of-band management. + * The check of the activity of the VMs on their volumes (KVMHAVMActivityChecker) is only done by Host HA, which needs it. + * + * If the host is checked on more than one storage pool, it is dead if its heartbeat is stopped on one of them when + * reportIfHeartBeatFailedForOneStoragePool is set, or else only if its heartbeat is stopped on all of them. */ @Override public Boolean hasHeartBeat() { @@ -44,16 +50,25 @@ public Boolean hasHeartBeat() { storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", "))); logger.debug("Checking heart beat with KVMHAChecker for {}", hostAndPools); + if (storagePools.isEmpty()) { + // this host monitors no storage pool, e.g. its agent is not upgraded yet and does not monitor the pools of the type + // which a newer version does: it has nothing to tell about the host, which is not a dead host + logger.debug("KVMHAChecker has no storage pool to check the heart beat for {}, so it can not tell whether the host is alive.", hostAndPools); + return true; + } + boolean heartBeatCheckResult = false; for (HAStoragePool pool : storagePools) { - heartBeatCheckResult = pool.getPool().hasHeartBeat(pool, host); - if (reportIfHeartBeatFailedForOneStoragePool && !heartBeatCheckResult) { + if (pool.getPool().hasHeartBeat(pool, host)) { + heartBeatCheckResult = true; + } else if (reportIfHeartBeatFailedForOneStoragePool) { + heartBeatCheckResult = false; break; } } if (!heartBeatCheckResult) { - logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools); + logger.warn("All checks with KVMHAChecker for {} considered it as dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostAndPools); } return heartBeatCheckResult; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index a5bea90903d0..48b14fd1dd99 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -20,6 +20,7 @@ import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.ha.HighAvailabilityManager; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; import org.libvirt.Connect; import org.libvirt.LibvirtException; import org.libvirt.StoragePool; @@ -38,9 +39,15 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; + private final String hostGuid; - public KVMHAMonitor(String host) { + /** + * @param host the private IP address of the host, which names its heartbeat on NFS and SharedMountPoint pools + * @param hostGuid the GUID of the host as the management server knows it, which names its heartbeat on RBD pools + */ + public KVMHAMonitor(String host, String hostGuid) { hostPrivateIp = host; + this.hostGuid = hostGuid; rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } @@ -60,7 +67,11 @@ public void removeStoragePool(String uuid) { synchronized (haStoragePools) { HAStoragePool pool = haStoragePools.get(uuid); if (pool != null) { - Script.runSimpleBashScript("umount " + pool.getMountDestPath()); + if (StringUtils.isNotEmpty(pool.getMountDestPath())) { + Script umount = new Script("umount", logger); + umount.add(pool.getMountDestPath()); + umount.execute(); + } haStoragePools.remove(uuid); } } @@ -94,7 +105,11 @@ protected void runHeartBeat() { if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { logger.warn("Write heartbeat for pool [{}] failed: {}; stopping cloudstack-agent.", uuid, result); - primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, null, false);; + try { + primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, null, false); + } catch (RuntimeException e) { + logger.error("Unable to stop the host after the failed heartbeat of pool [{}]", uuid, e); + } } } if (!removedPools.isEmpty()) { @@ -107,7 +122,15 @@ protected void runHeartBeat() { private String executePoolHeartBeatCommand(String uuid, HAStoragePool primaryStoragePool, String result) { for (int attempt = 1; attempt <= _heartBeatUpdateMaxTries; attempt++) { - result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); + try { + String heartBeatHostId = primaryStoragePool.getPool().getHeartBeatHostId(hostPrivateIp, hostGuid); + result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, heartBeatHostId, true); + } catch (RuntimeException e) { + // the heartbeat can not even be started, e.g. the pool is not configured completely or its script is missing: + // it is a failure of this pool only, which must not stop the heartbeats of the other pools, and trying again would not help + logger.error("Unable to run the heartbeat for pool [{}]: {}", uuid, e.toString(), e); + return "unable to run the heartbeat: " + e; + } if (result == null) { break; } @@ -136,7 +159,7 @@ private void checkForNotExistingLibvirtStoragePools(Set removedPools, St removedPools.add(uuid); } - logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid); + logger.debug("Found storage pool [{}] in libvirt, continuing.", uuid); } catch (LibvirtException e) { logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e); @@ -152,7 +175,12 @@ private void checkForNotExistingLibvirtStoragePools(Set removedPools, St public void run() { while (true) { - runHeartBeat(); + try { + runHeartBeat(); + } catch (RuntimeException e) { + // the thread must go on: if it stopped, the heartbeats of all the pools would stop and the host would look dead + logger.error("Unexpected error while writing the heartbeats", e); + } try { Thread.sleep(_heartBeatUpdateFreqInMs); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index acffa395494d..eb4a4c862cb7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1491,7 +1491,9 @@ public boolean configure(final String name, final Map params) th final String[] info = NetUtils.getNetworkParams(privateNic); - kvmhaMonitor = new KVMHAMonitor(info[0]); + // the GUID of the host as the management server knows it, as in Agent.getResourceGuid() + final String hostGuid = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.GUID) + "-" + name; + kvmhaMonitor = new KVMHAMonitor(info[0], hostGuid); final Thread haMonitorThread = new Thread(kvmhaMonitor); haMonitorThread.start(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index b132f6d98ce9..1506ac3b6d5c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -50,7 +50,15 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L if (primaryPool.isPoolSupportHA()) { final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid()); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); + if (haPool == null) { + // e.g. the heartbeat of the pool can not be used on this host, so the pool is not monitored for HA: it is not known whether the host is alive + return new Answer(command, false, "CheckVMActivityOnStoragePoolCommand: the storage pool " + pool.getUuid() + " is not monitored for HA on this host"); + } + final String vmActivityCheckPath = primaryPool.getVmActivityCheckPath(libvirtComputingResource.getVmActivityCheckPath()); + if (vmActivityCheckPath == null) { + return new Answer(command, false, "CheckVMActivityOnStoragePoolCommand: unable to find the script which checks the VM activity on a storage pool of type " + pool.getType()); + } + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 58665fdb5766..64aea9e8cee5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -155,6 +155,32 @@ default String getLocalPathFor(String relativePath) { String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation); + /** + * The id which names the heartbeat of a host on this storage pool: by default the private IP address of the host. + * + * @param hostPrivateIp the private IP address of the host + * @param hostGuid the GUID of the host as the management server knows it + */ + default String getHeartBeatHostId(String hostPrivateIp, String hostGuid) { + return hostPrivateIp; + } + + /** + * The script which checks the VM activity on this storage pool: by default the one given, which is the script of the + * pools with a shared mount point. Null if it can not be found. + */ + default String getVmActivityCheckPath(String defaultVmActivityCheckPath) { + return defaultVmActivityCheckPath; + } + + /** + * Returns why the heartbeat of this storage pool can not be used on this host, or null if it can. A host which can not + * write the heartbeat would be fenced, so such a pool is not monitored for HA. + */ + default String getHeartBeatPrerequisiteError() { + return null; + } + String getStorageNodeId(); Boolean hasHeartBeat(HAStoragePool pool, HostTO host); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 4a578ccf341d..64689507e648 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -37,6 +37,7 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.hypervisor.kvm.resource.KVMHABase; import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType; import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; @@ -428,8 +429,14 @@ private synchronized KVMStoragePool createStoragePool(String name, String host, // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { - KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, PoolType.PrimaryStorage); - _haMonitor.addStoragePool(storagePool); + String heartBeatPrerequisiteError = pool.getHeartBeatPrerequisiteError(); + if (heartBeatPrerequisiteError == null) { + KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, PoolType.PrimaryStorage); + _haMonitor.addStoragePool(storagePool); + } else { + // without the heartbeat the host would fail to write it at each cycle, and would be fenced + logger.error("The storage pool {} is not monitored for HA on this host, as its heartbeat can not be used: {}", pool.getUuid(), heartBeatPrerequisiteError); + } } StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, primaryStorage); addStoragePool(pool.getUuid(), info); @@ -445,7 +452,7 @@ public boolean disconnectPhysicalDisk(StoragePoolType type, String poolUuid, Str public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) { StorageAdaptor adaptor = getStorageAdaptor(type); - if (type == StoragePoolType.NetworkFilesystem) { + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) { _haMonitor.unregisterStoragePool(uuid); } boolean deleteStatus = adaptor.deleteStoragePool(uuid); @@ -458,7 +465,7 @@ public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); boolean deleteStatus; - if (type == StoragePoolType.NetworkFilesystem) { + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) { _haMonitor.unregisterStoragePool(uuid); deleteStatus = adaptor.deleteStoragePool(uuid); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index 7a70f2497cda..1a8bfd1f55b2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -18,10 +18,13 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.List; import java.util.Map; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joda.time.Duration; @@ -36,11 +39,17 @@ import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.RbdUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; public class LibvirtStoragePool implements KVMStoragePool { + /** The environment variable which gives the Ceph key to the RBD heartbeat and VM-activity scripts. */ + protected static final String RBD_KEY_ENVIRONMENT_VARIABLE = "CEPH_KEY"; + /** The time the RBD heartbeat script has to tell whether the Python bindings of librados and librbd are installed. */ + protected static final long BINDINGS_CHECK_TIMEOUT_IN_MS = 30000; + protected Logger logger = LogManager.getLogger(getClass()); protected String uuid; protected long capacity; @@ -357,37 +366,109 @@ public boolean isPoolSupportHA() { public String getHearthBeatPath() { if (StoragePoolType.NetworkFilesystem.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmheartbeat.sh"); } else if (StoragePoolType.SharedMountPoint.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmsmpheartbeat.sh"); + } else if (StoragePoolType.RBD.equals(type)) { + return findKvmHaScript("kvmheartbeat_rbd.py"); + } + return null; + } + + private String findKvmHaScript(String scriptName) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, scriptName); + if (scriptPath == null) { + throw new CloudRuntimeException(String.format("Unable to find script '%s' in directory: %s", scriptName, kvmScriptsDir)); + } + return scriptPath; + } + + /** + * Returns the environment of a heartbeat/VM-activity check script: for a RBD storage pool with cephx, the + * Ceph key, which is not an argument of the script as it would be visible in the process list. + */ + protected Map getHeartBeatScriptEnvironment() { + if (StoragePoolType.RBD.equals(type) && StringUtils.isNotBlank(authSecret)) { + return Map.of(RBD_KEY_ENVIRONMENT_VARIABLE, authSecret); } return null; } + /** + * Adds the Ceph cluster connection details (monitors, pool and, if cephx is enabled, the user; the key is in + * {@link #getHeartBeatScriptEnvironment()}) to a heartbeat/VM-activity check {@link Script} for a RBD storage pool. Mirrors the "mon_host"/"id"/"key" + * options that qemu itself uses to talk to RBD (see {@link KVMPhysicalDisk#RBDStringBuilder}). + */ + protected void addRbdConnectionArgs(Script cmd) { + boolean hasUser = StringUtils.isNotBlank(authUsername); + boolean hasSecret = StringUtils.isNotBlank(authSecret); + if (hasUser != hasSecret) { + throw new CloudRuntimeException(String.format("The Ceph user and key must be either both set or both unset for the storage pool %s", uuid)); + } + String monitors = StringUtils.isBlank(sourceHost) ? "" : RbdUtils.getRbdMonitors(sourceHost, sourcePort); + if (monitors.isEmpty()) { + throw new CloudRuntimeException(String.format("The storage pool %s has no Ceph monitors", uuid)); + } + cmd.add("-s", monitors); + cmd.add("-o", sourceDir); + if (hasUser) { + cmd.add("-n", authUsername); + } + } + + /** + * Adds the arguments identifying the storage to a heartbeat/VM-activity check {@link Script}: + * the Ceph connection details for a RBD pool, or the NFS server, path and mount point otherwise. + */ + private void addPoolConnectionArgs(Script cmd, HAStoragePool pool) { + if (StoragePoolType.RBD.equals(type)) { + addRbdConnectionArgs(cmd); + } else { + cmd.add("-i", pool.getPoolIp()); + cmd.add("-p", pool.getPoolMountSourcePath()); + cmd.add("-m", pool.getMountDestPath()); + } + } + + /** + * Adds the volumes of the host to a VM-activity check {@link Script}. For a RBD storage pool they are given in + * a file, as they can be too many for an argument of a command (the limit is 128 KiB, about 3500 volumes), + * which would also be written to the log, with all the volumes, at each check. Otherwise, or if the file can not + * be written, they are an argument. + * + * @return the file, which has to be deleted after the script ran, or null if there is none + */ + protected File addVolumeListArgs(Script cmd, String volumes) { + if (StoragePoolType.RBD.equals(type)) { + File volumeListFile = null; + try { + volumeListFile = File.createTempFile("kvmha-volumes-", ".txt"); + Files.write(volumeListFile.toPath(), volumes.getBytes(StandardCharsets.UTF_8)); + cmd.add("-U", volumeListFile.getAbsolutePath()); + return volumeListFile; + } catch (IOException e) { + logger.warn("Unable to write the list of volumes to a file, giving it to the script as an argument: {}", e.getMessage()); + if (volumeListFile != null && !volumeListFile.delete()) { + logger.debug("Unable to delete the file {}", volumeListFile); + } + } + } + cmd.add("-u", volumes); + return null; + } - public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { + public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String heartBeatHostId, boolean hostValidation) { Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger); - cmd.add("-i", primaryStoragePool.getPoolIp()); - cmd.add("-p", primaryStoragePool.getPoolMountSourcePath()); - cmd.add("-m", primaryStoragePool.getMountDestPath()); + addPoolConnectionArgs(cmd, primaryStoragePool); if (hostValidation) { - cmd.add("-h", hostPrivateIp); + cmd.add("-h", heartBeatHostId); } else { cmd.add("-c"); } - return cmd.execute(); + return cmd.execute(new OutputInterpreter.OutputLogger(logger), getHeartBeatScriptEnvironment()); } @Override @@ -400,54 +481,111 @@ public String getStorageNodeId() { return null; } + /** + * The heartbeat of a host on an RBD pool is named by the GUID of the host, which stays the same if the host + * gets a new IP address. On the other pool types it is named by the private IP address of the host. + */ + @Override + public String getHeartBeatHostId(String hostPrivateIp, String hostGuid) { + if (!StoragePoolType.RBD.equals(type)) { + return hostPrivateIp; + } + if (StringUtils.isBlank(hostGuid)) { + throw new CloudRuntimeException(String.format("The host %s has no GUID, which names its heartbeat on the RBD storage pool %s", hostPrivateIp, uuid)); + } + return hostGuid; + } + + protected String getHeartBeatHostId(HostTO host) { + return getHeartBeatHostId(host.getPrivateNetwork().getIp(), host.getGuid()); + } + + /** + * A RBD pool has no shared mount point to look at, so its script checks the watchers of the volumes, instead of the + * files of the volumes as the script of the other pools. + */ + @Override + public String getVmActivityCheckPath(String defaultVmActivityCheckPath) { + if (StoragePoolType.RBD.equals(type)) { + return Script.findScript(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR), "kvmvmactivity_rbd.py"); + } + return defaultVmActivityCheckPath; + } + + /** + * The RBD scripts need the Python bindings of librados and librbd, which the agent packages only suggest: a host + * without them can neither write its heartbeat (and would be fenced because of that), nor check the one of another host. + */ + @Override + public String getHeartBeatPrerequisiteError() { + if (!StoragePoolType.RBD.equals(type)) { + return null; + } + try { + Script cmd = new Script(getHearthBeatPath(), BINDINGS_CHECK_TIMEOUT_IN_MS, logger); + cmd.add("--check-bindings"); + // a result of the script is the error, which is its output + return cmd.execute(new OutputInterpreter.OutputLogger(logger)); + } catch (CloudRuntimeException e) { + return e.getMessage(); + } + } + @Override public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { - String hostIp = host.getPrivateNetwork().getIp(); + String heartBeatHostId = getHeartBeatHostId(host); Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); - cmd.add("-h", hostIp); + addPoolConnectionArgs(cmd, pool); + cmd.add("-h", heartBeatHostId); cmd.add("-r"); cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String result = cmd.execute(parser); + String result = cmd.execute(parser, getHeartBeatScriptEnvironment()); String parsedLine = parser.getLine(); - logger.debug("Checking heart beat for host IP {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); + logger.debug("Checking heart beat for host {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", heartBeatHostId, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + logger.warn("Checking heart beat for host {} with KVMHAChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", heartBeatHostId, cmd.toString(), parsedLine); return false; } else { - logger.debug("Checking heart beat for host IP {} with KVMHAChecker command [{}] succeeded.", hostIp, cmd.toString()); + logger.debug("Checking heart beat for host {} with KVMHAChecker command [{}] succeeded.", heartBeatHostId, cmd.toString()); return true; } } @Override public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { - String hostIp = host.getPrivateNetwork().getIp(); + String heartBeatHostId = getHeartBeatHostId(host); Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); - cmd.add("-h", hostIp); - cmd.add("-u", volumeUUIDListString); + addPoolConnectionArgs(cmd, pool); + cmd.add("-h", heartBeatHostId); + File volumeListFile = addVolumeListArgs(cmd, volumeUUIDListString); + if (StoragePoolType.RBD.equals(type)) { + // the age up to which a heartbeat counts as fresh depends on how often it is written + cmd.add("-i", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); + } cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000)); cmd.add("-d", String.valueOf(duration)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String result = cmd.execute(parser); + String result; + try { + result = cmd.execute(parser, getHeartBeatScriptEnvironment()); + } finally { + if (volumeListFile != null && !volumeListFile.delete()) { + logger.debug("Unable to delete the file {}", volumeListFile); + } + } String parsedLine = parser.getLine(); - logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); + logger.debug("Checking VM activity for host {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", heartBeatHostId, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + logger.warn("Checking VM activity for host {} with KVMHAVMActivityChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", heartBeatHostId, cmd.toString(), parsedLine); return false; } else { - logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] succeeded.", hostIp, cmd.toString()); + logger.debug("Checking VM activity for host {} with KVMHAVMActivityChecker command [{}] succeeded.", heartBeatHostId, cmd.toString()); return true; } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KVMHACheckerTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KVMHACheckerTest.java new file mode 100644 index 000000000000..39967b3b48ab --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KVMHACheckerTest.java @@ -0,0 +1,132 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.to.HostTO; +import com.cloud.agent.api.to.NetworkTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; +import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; + +@RunWith(MockitoJUnitRunner.class) +public class KVMHACheckerTest { + + private final HostTO host = mock(HostTO.class); + + @Before + public void setUp() { + NetworkTO network = mock(NetworkTO.class); + when(network.getIp()).thenReturn("10.0.0.1"); + when(host.getPrivateNetwork()).thenReturn(network); + } + + private KVMStoragePool mockPool(String uuid, boolean hasHeartBeat) { + KVMStoragePool pool = mock(KVMStoragePool.class); + when(pool.getUuid()).thenReturn(uuid); + lenient().when(pool.hasHeartBeat(any(), any())).thenReturn(hasHeartBeat); + return pool; + } + + private List haPools(KVMStoragePool... pools) { + List haPools = new ArrayList<>(); + for (KVMStoragePool pool : pools) { + haPools.add(new HAStoragePool(pool, "10.0.0.2", "/export", PoolType.PrimaryStorage)); + } + return haPools; + } + + private boolean hasHeartBeat(boolean reportIfHeartBeatFailedForOneStoragePool, KVMStoragePool... pools) { + return new KVMHAChecker(haPools(pools), host, reportIfHeartBeatFailedForOneStoragePool).hasHeartBeat(); + } + + @Test + public void testHostWithoutStoragePoolsToCheckIsNotDead() { + // e.g. a neighbour whose agent is not upgraded yet and monitors no pool: it can not tell, so it must not say that the host is dead + assertTrue(hasHeartBeat(true)); + assertTrue(hasHeartBeat(false)); + } + + @Test + public void testHostWithHeartBeatOnAllPoolsIsAlive() { + assertTrue(hasHeartBeat(true, mockPool("a", true), mockPool("b", true))); + assertTrue(hasHeartBeat(false, mockPool("a", true), mockPool("b", true))); + } + + @Test + public void testHostWithoutHeartBeatOnAllPoolsIsDead() { + assertFalse(hasHeartBeat(true, mockPool("a", false), mockPool("b", false))); + assertFalse(hasHeartBeat(false, mockPool("a", false), mockPool("b", false))); + } + + @Test + public void testHostIsDeadIfOnePoolHasNoHeartBeatAndThatIsReported() { + assertFalse(hasHeartBeat(true, mockPool("a", true), mockPool("b", false))); + assertFalse(hasHeartBeat(true, mockPool("a", false), mockPool("b", true))); + // the order of the pools does not matter, it was the last pool which decided + assertFalse(hasHeartBeat(true, mockPool("a", true), mockPool("b", false), mockPool("c", true))); + } + + @Test + public void testHostIsAliveIfOnePoolHasHeartBeatAndThatIsNotReported() { + // a dead heartbeat on the last pool, here the pool of another type, does not outvote an alive one on an other pool + assertTrue(hasHeartBeat(false, mockPool("a", true), mockPool("b", false))); + assertTrue(hasHeartBeat(false, mockPool("a", false), mockPool("b", true))); + assertTrue(hasHeartBeat(false, mockPool("a", false), mockPool("b", true), mockPool("c", false))); + } + + @Test + public void testPoolsAreNotCheckedAfterOneWithoutHeartBeatIsReported() { + KVMStoragePool first = mockPool("a", false); + KVMStoragePool second = mockPool("b", true); + + assertFalse(hasHeartBeat(true, first, second)); + + verify(second, never()).hasHeartBeat(any(), any()); + } + + @Test + public void testAllPoolsAreCheckedIfNotReported() { + List pools = Arrays.asList(mockPool("a", true), mockPool("b", false)); + + assertTrue(hasHeartBeat(false, pools.toArray(new KVMStoragePool[0]))); + + for (KVMStoragePool pool : pools) { + verify(pool).hasHeartBeat(any(), any()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitorTest.java new file mode 100644 index 000000000000..ea4c6524c425 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitorTest.java @@ -0,0 +1,130 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; +import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; +import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class KVMHAMonitorTest { + + private static final String PRIVATE_IP = "10.0.0.1"; + private static final String GUID = "7c5f1d70-1f36-4e3a-9b64-2d0d8a2f5f10-LibvirtComputingResource"; + + private final KVMHAMonitor monitor = new KVMHAMonitor(PRIVATE_IP, GUID); + + /** The type of the pools is not one with HA support, so that the monitor does not look for the pool in libvirt. */ + private KVMStoragePool addPool(String uuid) { + return addPool(monitor, uuid); + } + + private KVMStoragePool addPool(KVMHAMonitor haMonitor, String uuid) { + KVMStoragePool pool = mock(KVMStoragePool.class); + when(pool.getUuid()).thenReturn(uuid); + when(pool.getType()).thenReturn(StoragePoolType.CLVM); + haMonitor.addStoragePool(new HAStoragePool(pool, "10.0.0.2", "/export", PoolType.PrimaryStorage)); + return pool; + } + + @Test + public void testPoolNamesTheHeartBeatOfTheHost() { + KVMStoragePool pool = addPool("a"); + when(pool.getHeartBeatHostId(PRIVATE_IP, GUID)).thenReturn("the id of the host"); + when(pool.createHeartBeatCommand(any(), eq("the id of the host"), eq(true))).thenReturn(null); + + monitor.runHeartBeat(); + + verify(pool).createHeartBeatCommand(any(), eq("the id of the host"), eq(true)); + } + + @Test + public void testPoolWhichCanNotRunItsHeartBeatDoesNotStopTheOthers() { + KVMStoragePool broken = addPool("a"); + when(broken.getHeartBeatHostId(PRIVATE_IP, GUID)).thenThrow(new CloudRuntimeException("the host has no GUID")); + KVMStoragePool working = addPool("b"); + when(working.getHeartBeatHostId(PRIVATE_IP, GUID)).thenReturn(PRIVATE_IP); + when(working.createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true))).thenReturn(null); + + // it must not throw: the thread of the monitor would stop, and with it the heartbeats of all the pools + monitor.runHeartBeat(); + + verify(broken, never()).createHeartBeatCommand(any(), any(), eq(true)); + verify(working).createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true)); + } + + @Test + public void testPoolWhichThrowsWhileRunningItsHeartBeatDoesNotStopTheOthers() { + KVMStoragePool broken = addPool("a"); + when(broken.getHeartBeatHostId(PRIVATE_IP, GUID)).thenReturn(PRIVATE_IP); + when(broken.createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true))).thenThrow(new CloudRuntimeException("Unable to find script")); + KVMStoragePool working = addPool("b"); + when(working.getHeartBeatHostId(PRIVATE_IP, GUID)).thenReturn(PRIVATE_IP); + when(working.createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true))).thenReturn(null); + + monitor.runHeartBeat(); + + // a heartbeat which can not be run is not tried again, it would fail again + verify(broken, times(1)).createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true)); + verify(working).createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true)); + } + + @Test + public void testHostIsStoppedAfterAFailedHeartBeatAndAFailedStopDoesNotStopTheMonitor() { + try (MockedStatic properties = Mockito.mockStatic(AgentPropertiesFileHandler.class, Mockito.CALLS_REAL_METHODS)) { + properties.when(() -> AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT)).thenReturn(true); + KVMHAMonitor stoppingMonitor = new KVMHAMonitor(PRIVATE_IP, GUID); + + KVMStoragePool failing = addPool(stoppingMonitor, "a"); + when(failing.getHeartBeatHostId(PRIVATE_IP, GUID)).thenReturn(PRIVATE_IP); + // not a result of the script, which would be tried again at once and then each 10 seconds + when(failing.createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true))).thenThrow(new CloudRuntimeException("Unable to find script")); + when(failing.createHeartBeatCommand(any(), isNull(), eq(false))).thenThrow(new CloudRuntimeException("Unable to find script")); + KVMStoragePool working = addPool(stoppingMonitor, "b"); + when(working.getHeartBeatHostId(PRIVATE_IP, GUID)).thenReturn(PRIVATE_IP); + when(working.createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true))).thenReturn(null); + + stoppingMonitor.runHeartBeat(); + + verify(failing).createHeartBeatCommand(any(), isNull(), eq(false)); + verify(working).createHeartBeatCommand(any(), eq(PRIVATE_IP), eq(true)); + verify(working, never()).createHeartBeatCommand(any(), isNull(), eq(false)); + } + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 619aab27010e..c719ab5c0217 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -3199,7 +3199,8 @@ public void testCheckOnHostCommand() { final Answer answer = wrapper.execute(command, libvirtComputingResourceMock); assertTrue(answer.getResult()); assertTrue(answer instanceof CheckOnHostAnswer); - assertFalse(((CheckOnHostAnswer)answer).isAlive()); + // the host monitors no storage pool, so it can not tell: it must not say that the host is dead + assertTrue(((CheckOnHostAnswer)answer).isAlive()); verify(libvirtComputingResourceMock, times(1)).getMonitor(); } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java new file mode 100644 index 000000000000..ea5e3bbc9fbb --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java @@ -0,0 +1,175 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.joda.time.Duration; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; +import com.cloud.agent.api.to.HostTO; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase; +import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage.StoragePoolType; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest { + + private static final String POOL_UUID = "7684f6e5-fd91-3435-b04e-59ae0efb3914"; + private static final String RBD_SCRIPT = "/scripts/kvmvmactivity_rbd.py"; + private static final String SCRIPT = "/scripts/kvmvmactivity.sh"; + + private final LibvirtCheckVMActivityOnStoragePoolCommandWrapper wrapper = new LibvirtCheckVMActivityOnStoragePoolCommandWrapper(); + private final LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + private final KVMStoragePool pool = mock(KVMStoragePool.class); + private final CheckVMActivityOnStoragePoolCommand command = mock(CheckVMActivityOnStoragePoolCommand.class); + + private void setUp(StoragePoolType type, boolean supportsHa) { + StorageFilerTO filer = mock(StorageFilerTO.class); + when(filer.getType()).thenReturn(type); + when(filer.getUuid()).thenReturn(POOL_UUID); + when(command.getPool()).thenReturn(filer); + KVMStoragePoolManager poolManager = mock(KVMStoragePoolManager.class); + when(poolManager.getStoragePool(type, POOL_UUID)).thenReturn(pool); + when(resource.getStoragePoolMgr()).thenReturn(poolManager); + when(pool.isPoolSupportHA()).thenReturn(supportsHa); + } + + private KVMHABase.HAStoragePool addHaPool() { + KVMHABase.HAStoragePool haPool = new KVMHABase.HAStoragePool(pool, "10.0.0.1", "cloudstack", KVMHABase.PoolType.PrimaryStorage); + KVMHAMonitor monitor = mock(KVMHAMonitor.class); + when(monitor.getStoragePool(POOL_UUID)).thenReturn(haPool); + when(resource.getMonitor()).thenReturn(monitor); + return haPool; + } + + private void expectActivityCheck(String volumes, long suspectTime, boolean activity) { + when(command.getHost()).thenReturn(mock(HostTO.class)); + when(command.getVolumeList()).thenReturn(volumes); + when(command.getSuspectTimeInSeconds()).thenReturn(suspectTime); + when(pool.hasVmActivity(any(), any(), any(), anyString(), anyString(), anyLong())).thenReturn(activity); + } + + @Test + public void testRbdPoolIsCheckedWithTheScriptOfTheRbdPool() { + setUp(StoragePoolType.RBD, true); + KVMHABase.HAStoragePool haPool = addHaPool(); + expectActivityCheck("volume-1,volume-2", 123L, false); + when(resource.getVmActivityCheckPath()).thenReturn(SCRIPT); + // the pool owns which script checks it, whatever the script of the agent is + when(pool.getVmActivityCheckPath(SCRIPT)).thenReturn(RBD_SCRIPT); + + Answer answer = wrapper.execute(command, resource); + + assertTrue(answer.getResult()); + verify(pool).hasVmActivity(eq(haPool), any(HostTO.class), any(Duration.class), eq("volume-1,volume-2"), eq(RBD_SCRIPT), eq(123L)); + } + + @Test + public void testOtherPoolsAreCheckedWithTheScriptOfTheAgent() { + for (StoragePoolType type : new StoragePoolType[] {StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint}) { + Mockito.reset(resource, pool, command); + setUp(type, true); + KVMHABase.HAStoragePool haPool = addHaPool(); + expectActivityCheck("volume-1", 5L, false); + when(resource.getVmActivityCheckPath()).thenReturn(SCRIPT); + when(pool.getVmActivityCheckPath(SCRIPT)).thenReturn(SCRIPT); + + Answer answer = wrapper.execute(command, resource); + + assertTrue(answer.getResult()); + verify(pool).hasVmActivity(eq(haPool), any(HostTO.class), any(Duration.class), eq("volume-1"), eq(SCRIPT), eq(5L)); + } + } + + @Test + public void testActivityIsReported() { + setUp(StoragePoolType.NetworkFilesystem, true); + addHaPool(); + expectActivityCheck("volume-1", 5L, true); + when(resource.getVmActivityCheckPath()).thenReturn(SCRIPT); + when(pool.getVmActivityCheckPath(SCRIPT)).thenReturn(SCRIPT); + + Answer answer = wrapper.execute(command, resource); + + assertFalse(answer.getResult()); + assertEquals("VMHA disk activity detected ...", answer.getDetails()); + } + + @Test + public void testMissingScriptOfThePoolIsAnError() { + setUp(StoragePoolType.RBD, true); + addHaPool(); + when(pool.getVmActivityCheckPath(any())).thenReturn(null); + + Answer answer = wrapper.execute(command, resource); + + assertFalse(answer.getResult()); + assertTrue(answer.getDetails().contains("unable to find the script")); + verify(pool, never()).hasVmActivity(any(), any(), any(), any(), any(), anyLong()); + } + + @Test + public void testPoolWhichIsNotMonitoredForHaOnTheHostIsNotChecked() { + setUp(StoragePoolType.RBD, true); + // e.g. the Python bindings of Ceph are not installed, so the pool was not registered with the HA monitor + KVMHAMonitor monitor = mock(KVMHAMonitor.class); + when(monitor.getStoragePool(POOL_UUID)).thenReturn(null); + when(resource.getMonitor()).thenReturn(monitor); + + Answer answer = wrapper.execute(command, resource); + + // not a answer which says that the host has no activity + assertFalse(answer.getResult()); + assertTrue(answer.getDetails().contains("is not monitored for HA on this host")); + verify(pool, never()).hasVmActivity(any(), any(), any(), any(), any(), anyLong()); + } + + @Test + public void testPoolWithoutHaSupportIsNotChecked() { + setUp(StoragePoolType.RBD, false); + when(resource.getMonitor()).thenReturn(mock(KVMHAMonitor.class)); + + Answer answer = wrapper.execute(command, resource); + + assertFalse(answer.getResult()); + assertEquals("Unsupported Storage", answer.getDetails()); + verify(pool, never()).hasVmActivity(any(), any(), any(), any(), any(), anyLong()); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManagerTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManagerTest.java new file mode 100644 index 000000000000..7de792f58132 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManagerTest.java @@ -0,0 +1,145 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.storage; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.withSettings; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.logging.log4j.LogManager; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; +import com.cloud.storage.Storage.StoragePoolType; + +@RunWith(MockitoJUnitRunner.class) +public class KVMStoragePoolManagerTest { + + private static final String POOL_UUID = "7684f6e5-fd91-3435-b04e-59ae0efb3914"; + + private final KVMHAMonitor monitor = mock(KVMHAMonitor.class); + private final StorageAdaptor adaptor = mock(StorageAdaptor.class); + private final Map details = new HashMap<>(); + private KVMStoragePoolManager manager; + + @Before + public void setUp() throws Exception { + // the constructor is not run, as it looks for all the storage adaptors of the agent, some of which need scripts and libraries of a host + manager = mock(KVMStoragePoolManager.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); + // every type of pool gets the adaptor of the test + Map adaptors = new HashMap<>(); + adaptors.put("libvirt", adaptor); + setField("_storageMapper", adaptors); + setField("_storagePools", new ConcurrentHashMap()); + setField("_haMonitor", monitor); + setField("logger", LogManager.getLogger(KVMStoragePoolManager.class)); + } + + private void setField(String name, Object value) throws Exception { + Field field = KVMStoragePoolManager.class.getDeclaredField(name); + field.setAccessible(true); + field.set(manager, value); + } + + @Test + public void testPoolsWithHaSupportAreRemovedFromTheMonitorWhenDeleted() { + for (StoragePoolType type : new StoragePoolType[] {StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint, StoragePoolType.RBD}) { + when(adaptor.deleteStoragePool(POOL_UUID)).thenReturn(true); + when(adaptor.deleteStoragePool(POOL_UUID, details)).thenReturn(true); + + assertTrue(manager.deleteStoragePool(type, POOL_UUID)); + assertTrue(manager.deleteStoragePool(type, POOL_UUID, details)); + } + verify(monitor, times(6)).removeStoragePool(POOL_UUID); + } + + @Test + public void testPoolsAreDeletedWithDetailsAsBefore() { + // the delete which takes the details does not destroy the libvirt pool, which the VMs may still use, e.g. in maintenance of the pool + for (StoragePoolType type : new StoragePoolType[] {StoragePoolType.NetworkFilesystem, StoragePoolType.RBD, StoragePoolType.SharedMountPoint}) { + reset(adaptor, monitor); + when(adaptor.deleteStoragePool(POOL_UUID, details)).thenReturn(true); + + assertTrue(manager.deleteStoragePool(type, POOL_UUID, details)); + + verify(monitor).removeStoragePool(POOL_UUID); + verify(adaptor).deleteStoragePool(POOL_UUID, details); + verify(adaptor, never()).deleteStoragePool(POOL_UUID); + } + } + + @Test + public void testPoolWithoutHaSupportIsNotRemovedFromTheMonitor() { + when(adaptor.deleteStoragePool(POOL_UUID, details)).thenReturn(true); + + assertTrue(manager.deleteStoragePool(StoragePoolType.CLVM, POOL_UUID, details)); + + verify(monitor, never()).removeStoragePool(any()); + verify(adaptor, never()).deleteStoragePool(POOL_UUID); + } + + private KVMStoragePool mockCreatedPool(String prerequisiteError) { + KVMStoragePool pool = mock(KVMStoragePool.class); + when(pool.getUuid()).thenReturn(POOL_UUID); + when(pool.isPoolSupportHA()).thenReturn(true); + when(pool.getHeartBeatPrerequisiteError()).thenReturn(prerequisiteError); + when(adaptor.createStoragePool(any(), any(), anyInt(), any(), any(), any(), any(), eq(true))).thenReturn(pool); + return pool; + } + + @Test + public void testPoolWhichHeartBeatCanBeUsedIsMonitoredForHa() { + mockCreatedPool(null); + + manager.createStoragePool("pool", "10.0.0.1", 6789, "rbdpool", "cephuser:cephkey", StoragePoolType.RBD, details); + + verify(monitor).addStoragePool(any()); + } + + @Test + public void testPoolWhichHeartBeatCanNotBeUsedIsNotMonitoredForHa() { + // e.g. the Python bindings of Ceph are not installed: the host could not write its heartbeat, and would be fenced + mockCreatedPool("the package python3-rados is not installed"); + + KVMStoragePool pool = manager.createStoragePool("pool", "10.0.0.1", 6789, "rbdpool", "cephuser:cephkey", StoragePoolType.RBD, details); + + verify(monitor, never()).addStoragePool(any()); + // the pool itself is still usable + assertNotNull(pool); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePoolTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePoolTest.java index ea5df8e741c4..6f6041407d84 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePoolTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePoolTest.java @@ -23,9 +23,16 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.libvirt.StoragePool; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import com.cloud.agent.api.to.HostTO; +import com.cloud.agent.api.to.NetworkTO; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; import junit.framework.TestCase; import org.mockito.junit.MockitoJUnitRunner; @@ -102,6 +109,132 @@ public void testExternalSnapshot() { assertTrue(clvmPool.isExternalSnapshot()); } + @Test + public void testIsPoolSupportHA() { + String uuid = "0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5"; + String name = "myfirstpool"; + + StorageAdaptor adapter = Mockito.mock(LibvirtStorageAdaptor.class); + StoragePool storage = Mockito.mock(StoragePool.class); + + // NetworkFilesystem, SharedMountPoint and RBD all support the KVM Host-HA + // heartbeat/VM-activity check mechanism. + assertTrue(new LibvirtStoragePool(uuid, name, StoragePoolType.NetworkFilesystem, adapter, storage).isPoolSupportHA()); + assertTrue(new LibvirtStoragePool(uuid, name, StoragePoolType.SharedMountPoint, adapter, storage).isPoolSupportHA()); + assertTrue(new LibvirtStoragePool(uuid, name, StoragePoolType.RBD, adapter, storage).isPoolSupportHA()); + + // Other pool types have no HA support. + assertFalse(new LibvirtStoragePool(uuid, name, StoragePoolType.CLVM, adapter, storage).isPoolSupportHA()); + assertFalse(new LibvirtStoragePool(uuid, name, StoragePoolType.Filesystem, adapter, storage).isPoolSupportHA()); + } + + private LibvirtStoragePool getRbdPool(String authUsername, String authSecret) { + LibvirtStoragePool pool = new LibvirtStoragePool("0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5", "myfirstpool", StoragePoolType.RBD, + Mockito.mock(LibvirtStorageAdaptor.class), Mockito.mock(StoragePool.class)); + pool.setSourceHost("10.0.0.1,fd00::1"); + pool.setSourcePort(6789); + pool.setSourceDir("rbdpool"); + pool.setAuthUsername(authUsername); + pool.setAuthSecret(authSecret); + return pool; + } + + @Test + public void testAddRbdConnectionArgsWithCephx() { + Script cmd = Mockito.mock(Script.class); + getRbdPool("cephuser", "cephkey").addRbdConnectionArgs(cmd); + Mockito.verify(cmd).add("-s", "10.0.0.1:6789,[fd00::1]:6789"); + Mockito.verify(cmd).add("-o", "rbdpool"); + Mockito.verify(cmd).add("-n", "cephuser"); + // the key is not an argument of the script, it would be visible in the process list + Mockito.verifyNoMoreInteractions(cmd); + } + + @Test + public void testAddRbdConnectionArgsWithoutCephx() { + for (String[] noAuth : new String[][] {{null, null}, {"", ""}, {" ", null}}) { + Script cmd = Mockito.mock(Script.class); + getRbdPool(noAuth[0], noAuth[1]).addRbdConnectionArgs(cmd); + Mockito.verify(cmd).add("-s", "10.0.0.1:6789,[fd00::1]:6789"); + Mockito.verify(cmd).add("-o", "rbdpool"); + Mockito.verifyNoMoreInteractions(cmd); + } + } + + @Test + public void testAddRbdConnectionArgsWithCephUserOnlyOrKeyOnly() { + for (String[] partial : new String[][] {{"cephuser", null}, {"cephuser", " "}, {null, "cephkey"}, {"", "cephkey"}}) { + Script cmd = Mockito.mock(Script.class); + try { + getRbdPool(partial[0], partial[1]).addRbdConnectionArgs(cmd); + fail("Expected a CloudRuntimeException for user [" + partial[0] + "] and key [" + partial[1] + "]"); + } catch (CloudRuntimeException expected) { + Mockito.verifyNoInteractions(cmd); + } + } + } + + @Test + public void testHeartBeatScriptEnvironmentHasTheCephKey() { + assertEquals(java.util.Map.of("CEPH_KEY", "cephkey"), getRbdPool("cephuser", "cephkey").getHeartBeatScriptEnvironment()); + } + + @Test + public void testHeartBeatScriptEnvironmentWithoutCephx() { + assertNull(getRbdPool(null, null).getHeartBeatScriptEnvironment()); + assertNull(getRbdPool("", " ").getHeartBeatScriptEnvironment()); + } + + @Test + public void testHeartBeatScriptEnvironmentOfOtherPoolTypes() { + LibvirtStoragePool pool = new LibvirtStoragePool("0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5", "myfirstpool", StoragePoolType.NetworkFilesystem, + Mockito.mock(LibvirtStorageAdaptor.class), Mockito.mock(StoragePool.class)); + pool.setAuthSecret("cephkey"); + assertNull(pool.getHeartBeatScriptEnvironment()); + } + + @Test + public void testVolumeListOfRbdPoolIsGivenInAFile() throws java.io.IOException { + Script cmd = Mockito.mock(Script.class); + String volumes = "uuid-1,uuid-2,uuid-3"; + java.io.File file = getRbdPool("cephuser", "cephkey").addVolumeListArgs(cmd, volumes); + try { + assertNotNull(file); + assertEquals(volumes, new String(java.nio.file.Files.readAllBytes(file.toPath()), java.nio.charset.StandardCharsets.UTF_8)); + Mockito.verify(cmd).add("-U", file.getAbsolutePath()); + Mockito.verifyNoMoreInteractions(cmd); + } finally { + if (file != null) { + file.delete(); + } + } + } + + @Test + public void testVolumeListOfRbdPoolCanBeVeryLong() throws java.io.IOException { + // more than what an argument of a command can have (128 KiB) + String volumes = String.join(",", java.util.Collections.nCopies(10000, "6d93a798-9fbe-41e2-8b69-fc6b5782f89a")); + java.io.File file = getRbdPool("cephuser", "cephkey").addVolumeListArgs(Mockito.mock(Script.class), volumes); + try { + assertNotNull(file); + assertEquals(volumes.length(), file.length()); + } finally { + if (file != null) { + file.delete(); + } + } + } + + @Test + public void testVolumeListOfOtherPoolTypesIsAnArgument() { + Script cmd = Mockito.mock(Script.class); + LibvirtStoragePool pool = new LibvirtStoragePool("0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5", "myfirstpool", StoragePoolType.NetworkFilesystem, + Mockito.mock(LibvirtStorageAdaptor.class), Mockito.mock(StoragePool.class)); + assertNull(pool.addVolumeListArgs(cmd, "uuid-1,uuid-2")); + Mockito.verify(cmd).add("-u", "uuid-1,uuid-2"); + Mockito.verifyNoMoreInteractions(cmd); + } + @Test public void testResolvePhysicalDiskFileUsesNestedRelativePathInsidePool() throws Exception { File poolPath = Files.createTempDirectory("libvirt-storage-pool").toFile(); @@ -154,4 +287,143 @@ private void deleteRecursively(File file) { } file.delete(); } + + private HostTO getHost(String ip, String guid) { + HostTO host = Mockito.mock(HostTO.class); + NetworkTO privateNetwork = new NetworkTO(); + privateNetwork.setIp(ip); + Mockito.when(host.getGuid()).thenReturn(guid); + Mockito.when(host.getPrivateNetwork()).thenReturn(privateNetwork); + return host; + } + + private LibvirtStoragePool getPool(StoragePoolType type) { + return new LibvirtStoragePool("0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5", "myfirstpool", type, + Mockito.mock(LibvirtStorageAdaptor.class), Mockito.mock(StoragePool.class)); + } + + @Test + public void testHeartBeatHostIdOfRbdIsTheGuid() { + HostTO host = getHost("10.0.0.5", "5d3a7b0e-1c2f-4a8e-9b61-0c9e1f4d2a77-LibvirtComputingResource"); + assertEquals("5d3a7b0e-1c2f-4a8e-9b61-0c9e1f4d2a77-LibvirtComputingResource", getPool(StoragePoolType.RBD).getHeartBeatHostId(host)); + } + + @Test + public void testHeartBeatHostIdOfOtherPoolsIsTheIp() { + HostTO host = getHost("10.0.0.5", "5d3a7b0e-1c2f-4a8e-9b61-0c9e1f4d2a77-LibvirtComputingResource"); + assertEquals("10.0.0.5", getPool(StoragePoolType.NetworkFilesystem).getHeartBeatHostId(host)); + assertEquals("10.0.0.5", getPool(StoragePoolType.SharedMountPoint).getHeartBeatHostId(host)); + } + + @Test(expected = CloudRuntimeException.class) + public void testHeartBeatHostIdOfRbdWithoutGuid() { + getPool(StoragePoolType.RBD).getHeartBeatHostId(getHost("10.0.0.5", " ")); + } + + @Test + public void testHeartBeatHostIdOfOtherPoolsDoesNotNeedAGuid() { + assertEquals("10.0.0.5", getPool(StoragePoolType.NetworkFilesystem).getHeartBeatHostId(getHost("10.0.0.5", null))); + } + + @Test + public void testPoolNamesTheHeartBeatOfTheHostWithItsGuidOrItsIp() { + String guid = "5d3a7b0e-1c2f-4a8e-9b61-0c9e1f4d2a77-LibvirtComputingResource"; + assertEquals(guid, getPool(StoragePoolType.RBD).getHeartBeatHostId("10.0.0.5", guid)); + assertEquals("10.0.0.5", getPool(StoragePoolType.NetworkFilesystem).getHeartBeatHostId("10.0.0.5", guid)); + assertEquals("10.0.0.5", getPool(StoragePoolType.SharedMountPoint).getHeartBeatHostId("10.0.0.5", null)); + } + + @Test + public void testPoolCanNotNameTheHeartBeatOfARbdHostWithoutGuid() { + for (String guid : new String[] {null, "", " "}) { + try { + getPool(StoragePoolType.RBD).getHeartBeatHostId("10.0.0.5", guid); + fail("Expected a CloudRuntimeException for the GUID [" + guid + "]"); + } catch (CloudRuntimeException expected) { + assertTrue(expected.getMessage().contains("has no GUID")); + } + } + } + + @Test + public void testVmActivityCheckPathOfOtherPoolsIsTheOneGiven() { + assertEquals("/scripts/kvmvmactivity.sh", getPool(StoragePoolType.NetworkFilesystem).getVmActivityCheckPath("/scripts/kvmvmactivity.sh")); + assertEquals("/scripts/kvmvmactivity.sh", getPool(StoragePoolType.SharedMountPoint).getVmActivityCheckPath("/scripts/kvmvmactivity.sh")); + assertNull(getPool(StoragePoolType.NetworkFilesystem).getVmActivityCheckPath(null)); + } + + @Test + public void testVmActivityCheckPathOfRbdIsItsOwnScript() { + try (MockedStatic