From 15eb8392f96ccf3cb0a754ae84d49864fe3d07e7 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 22 Jul 2026 15:03:56 +0200 Subject: [PATCH 1/3] add an maintenance prepare error event --- api/src/main/java/com/cloud/event/EventTypes.java | 2 ++ .../com/cloud/resource/ResourceManagerImpl.java | 3 +++ .../com/cloud/resource/ResourceManagerImplTest.java | 13 +++++++++++++ 3 files changed, 18 insertions(+) diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 81ed185dae5a..3ef7db9ea82d 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -458,6 +458,7 @@ public class EventTypes { public static final String EVENT_MAINTENANCE_CANCEL = "MAINT.CANCEL"; public static final String EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE = "MAINT.CANCEL.PS"; public static final String EVENT_MAINTENANCE_PREPARE = "MAINT.PREPARE"; + public static final String EVENT_MAINTENANCE_PREPARE_ERROR = "MAINT.PREPARE.ERROR"; public static final String EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE = "MAINT.PREPARE.PS"; // Primary storage pool @@ -1046,6 +1047,7 @@ public class EventTypes { entityEventDetails.put(EVENT_MAINTENANCE_CANCEL, Host.class); entityEventDetails.put(EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE, Host.class); entityEventDetails.put(EVENT_MAINTENANCE_PREPARE, Host.class); + entityEventDetails.put(EVENT_MAINTENANCE_PREPARE_ERROR, Host.class); entityEventDetails.put(EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE, Host.class); // Primary storage pool diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index a6a75fc39613..e0185a10c4f3 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1730,6 +1730,9 @@ protected boolean setHostIntoErrorInPrepareForMaintenance(HostVO host, List ActionEventUtils.onCompletedActionEvent( + anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR), + eq(expectedDescription), eq(hostId), isNull(), eq(0L))); Assert.assertFalse(enterMaintenanceMode); } + @Test + public void testMaintenancePrepareErrorEventMapsToHostEntity() { + Assert.assertEquals(Host.class, EventTypes.getEntityClassForEvent(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR)); + } + private void verifyReturnToPrepareForMaintenanceCalls() throws NoTransitionException { boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId); verify(resourceManager).attemptMaintain(host); From ad045256f0b999a3ae8916dd43ad0921fe433a24 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 21 Oct 2026 09:23:31 +0200 Subject: [PATCH 2/3] emit error event on ErrorInMaintenance, not just ErrorInPrepareForMaintenance --- .../src/main/java/com/cloud/resource/ResourceManagerImpl.java | 3 +++ .../test/java/com/cloud/resource/ResourceManagerImplTest.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index e0185a10c4f3..113ea3949857 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1723,6 +1723,9 @@ protected boolean setHostIntoErrorInMaintenance(HostVO host, List _haMgr.cancelScheduledMigrations(host); configureVncAccessForKVMHostFailedMigrations(host, errorVms); resourceStateTransitTo(host, ResourceState.Event.UnableToMaintain, _nodeId); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_ERROR, EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR, + String.format("failed to put host %s into maintenance due to running VMs that could not be migrated or VMs in error state", host), host.getId(), null, 0); return false; } diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 5bf924ef34dd..32d98cc62984 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -524,6 +524,7 @@ private void setupErrorVms() { } private void verifyErrorInMaintenanceCalls() throws NoTransitionException { + when(host.getResourceState()).thenReturn(ResourceState.PrepareForMaintenance); boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId); verify(resourceManager).attemptMaintain(host); verify(resourceManager).setHostIntoErrorInMaintenance(eq(host), any()); @@ -531,6 +532,9 @@ private void verifyErrorInMaintenanceCalls() throws NoTransitionException { verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any()); verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any()); verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMaintain), anyLong()); + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent( + anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR), + anyString(), eq(hostId), isNull(), eq(0L))); Assert.assertFalse(enterMaintenanceMode); } From 6036002f3f067f60b995ea3b9a0ea11bd004d630 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Tue, 3 Nov 2026 10:50:42 +0100 Subject: [PATCH 3/3] emit maintenance prepare error event only when entering ErrorInPrepareForMaintenance --- .../cloud/resource/ResourceManagerImpl.java | 21 +++++++++++++------ .../resource/ResourceManagerImplTest.java | 14 +++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 113ea3949857..1b75b93ae7b5 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1723,22 +1723,31 @@ protected boolean setHostIntoErrorInMaintenance(HostVO host, List _haMgr.cancelScheduledMigrations(host); configureVncAccessForKVMHostFailedMigrations(host, errorVms); resourceStateTransitTo(host, ResourceState.Event.UnableToMaintain, _nodeId); - ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), - EventVO.LEVEL_ERROR, EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR, - String.format("failed to put host %s into maintenance due to running VMs that could not be migrated or VMs in error state", host), host.getId(), null, 0); + emitMaintenanceErrorEvent(host, String.format("failed to put host %s into maintenance due to running VMs that could not be migrated or VMs in error state", host)); return false; } + /** + * Set host into ErrorInPrepareForMaintenance state. The agent monitor calls this on every check while the errors + * persist, so the error event is only emitted when the host enters this state. + */ protected boolean setHostIntoErrorInPrepareForMaintenance(HostVO host, List errorVms) throws NoTransitionException { logger.debug("Host {} entering in PrepareForMaintenanceWithErrors state", host); + final boolean alreadyInErrorState = host.getResourceState() == ResourceState.ErrorInPrepareForMaintenance; configureVncAccessForKVMHostFailedMigrations(host, errorVms); resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId); - ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), - EventVO.LEVEL_ERROR, EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR, - String.format("failed to prepare host %s for maintenance due to migration or VM state errors", host), host.getId(), null, 0); + if (!alreadyInErrorState) { + emitMaintenanceErrorEvent(host, String.format("failed to prepare host %s for maintenance due to migration or VM state errors", host)); + } return false; } + private void emitMaintenanceErrorEvent(HostVO host, String description) { + final CallContext context = CallContext.current(); + ActionEventUtils.onCompletedActionEvent(context.getCallingUserId(), context.getCallingAccountId(), + EventVO.LEVEL_ERROR, EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR, description, host.getId(), null, 0); + } + protected boolean setHostIntoPrepareForMaintenanceAfterErrorsFixed(HostVO host) throws NoTransitionException { logger.debug("Host {} entering in PrepareForMaintenance state as any previous corrections have been fixed", host); resourceStateTransitTo(host, ResourceState.Event.ErrorsCorrected, _nodeId); diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 32d98cc62984..43e8eb1db351 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -554,6 +554,20 @@ private void verifyErrorInPrepareForMaintenanceCalls() throws NoTransitionExcept Assert.assertFalse(enterMaintenanceMode); } + @Test + public void testCheckAndMaintainStillInErrorInPrepareForMaintenanceDoesNotRepeatErrorEvent() throws NoTransitionException { + setupFailedMigrations(); + setupPendingMigrationRetries(); + when(host.getResourceState()).thenReturn(ResourceState.ErrorInPrepareForMaintenance); + boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId); + verify(resourceManager).setHostIntoErrorInPrepareForMaintenance(eq(host), any()); + verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong()); + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent( + anyLong(), anyLong(), anyString(), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR), + anyString(), anyLong(), any(), anyLong()), never()); + Assert.assertFalse(enterMaintenanceMode); + } + @Test public void testMaintenancePrepareErrorEventMapsToHostEntity() { Assert.assertEquals(Host.class, EventTypes.getEntityClassForEvent(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR));