diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 81ed185dae5a..3ef7db9ea82d 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -458,6 +458,7 @@ public class EventTypes { public static final String EVENT_MAINTENANCE_CANCEL = "MAINT.CANCEL"; public static final String EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE = "MAINT.CANCEL.PS"; public static final String EVENT_MAINTENANCE_PREPARE = "MAINT.PREPARE"; + public static final String EVENT_MAINTENANCE_PREPARE_ERROR = "MAINT.PREPARE.ERROR"; public static final String EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE = "MAINT.PREPARE.PS"; // Primary storage pool @@ -1046,6 +1047,7 @@ public class EventTypes { entityEventDetails.put(EVENT_MAINTENANCE_CANCEL, Host.class); entityEventDetails.put(EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE, Host.class); entityEventDetails.put(EVENT_MAINTENANCE_PREPARE, Host.class); + entityEventDetails.put(EVENT_MAINTENANCE_PREPARE_ERROR, Host.class); entityEventDetails.put(EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE, Host.class); // Primary storage pool diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index a6a75fc39613..1b75b93ae7b5 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1723,16 +1723,31 @@ protected boolean setHostIntoErrorInMaintenance(HostVO host, List _haMgr.cancelScheduledMigrations(host); configureVncAccessForKVMHostFailedMigrations(host, errorVms); resourceStateTransitTo(host, ResourceState.Event.UnableToMaintain, _nodeId); + emitMaintenanceErrorEvent(host, String.format("failed to put host %s into maintenance due to running VMs that could not be migrated or VMs in error state", host)); return false; } + /** + * Set host into ErrorInPrepareForMaintenance state. The agent monitor calls this on every check while the errors + * persist, so the error event is only emitted when the host enters this state. + */ protected boolean setHostIntoErrorInPrepareForMaintenance(HostVO host, List errorVms) throws NoTransitionException { logger.debug("Host {} entering in PrepareForMaintenanceWithErrors state", host); + final boolean alreadyInErrorState = host.getResourceState() == ResourceState.ErrorInPrepareForMaintenance; configureVncAccessForKVMHostFailedMigrations(host, errorVms); resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId); + if (!alreadyInErrorState) { + emitMaintenanceErrorEvent(host, String.format("failed to prepare host %s for maintenance due to migration or VM state errors", host)); + } return false; } + private void emitMaintenanceErrorEvent(HostVO host, String description) { + final CallContext context = CallContext.current(); + ActionEventUtils.onCompletedActionEvent(context.getCallingUserId(), context.getCallingAccountId(), + EventVO.LEVEL_ERROR, EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR, description, host.getId(), null, 0); + } + protected boolean setHostIntoPrepareForMaintenanceAfterErrorsFixed(HostVO host) throws NoTransitionException { logger.debug("Host {} entering in PrepareForMaintenance state as any previous corrections have been fixed", host); resourceStateTransitTo(host, ResourceState.Event.ErrorsCorrected, _nodeId); diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index a28d3aaeac07..43e8eb1db351 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -23,6 +23,8 @@ import com.cloud.agent.api.GetVncPortCommand; import com.cloud.capacity.dao.CapacityDao; import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; import com.cloud.exception.InvalidParameterValueException; import com.cloud.ha.HighAvailabilityManager; import com.cloud.host.Host; @@ -79,6 +81,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -521,6 +524,7 @@ private void setupErrorVms() { } private void verifyErrorInMaintenanceCalls() throws NoTransitionException { + when(host.getResourceState()).thenReturn(ResourceState.PrepareForMaintenance); boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId); verify(resourceManager).attemptMaintain(host); verify(resourceManager).setHostIntoErrorInMaintenance(eq(host), any()); @@ -528,10 +532,14 @@ private void verifyErrorInMaintenanceCalls() throws NoTransitionException { verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any()); verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any()); verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMaintain), anyLong()); + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent( + anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR), + anyString(), eq(hostId), isNull(), eq(0L))); Assert.assertFalse(enterMaintenanceMode); } private void verifyErrorInPrepareForMaintenanceCalls() throws NoTransitionException { + when(host.getResourceState()).thenReturn(ResourceState.PrepareForMaintenance); boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId); verify(resourceManager).attemptMaintain(host); verify(resourceManager).setHostIntoErrorInPrepareForMaintenance(eq(host), any()); @@ -539,9 +547,32 @@ private void verifyErrorInPrepareForMaintenanceCalls() throws NoTransitionExcept verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any()); verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any()); verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong()); + String expectedDescription = String.format("failed to prepare host %s for maintenance due to migration or VM state errors", host); + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent( + anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR), + eq(expectedDescription), eq(hostId), isNull(), eq(0L))); Assert.assertFalse(enterMaintenanceMode); } + @Test + public void testCheckAndMaintainStillInErrorInPrepareForMaintenanceDoesNotRepeatErrorEvent() throws NoTransitionException { + setupFailedMigrations(); + setupPendingMigrationRetries(); + when(host.getResourceState()).thenReturn(ResourceState.ErrorInPrepareForMaintenance); + boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId); + verify(resourceManager).setHostIntoErrorInPrepareForMaintenance(eq(host), any()); + verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong()); + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent( + anyLong(), anyLong(), anyString(), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR), + anyString(), anyLong(), any(), anyLong()), never()); + Assert.assertFalse(enterMaintenanceMode); + } + + @Test + public void testMaintenancePrepareErrorEventMapsToHostEntity() { + Assert.assertEquals(Host.class, EventTypes.getEntityClassForEvent(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR)); + } + private void verifyReturnToPrepareForMaintenanceCalls() throws NoTransitionException { boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId); verify(resourceManager).attemptMaintain(host);