Skip to content

add a maintenance prepare error event #13669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
DaanHoogland wants to merge 3 commits into
base: 4.20
Choose a base branch
from
Open

add a maintenance prepare error event #13669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
15eb839
add an maintenance prepare error event
DaanHoogland Jul 22, 2026
ad04525
emit error event on ErrorInMaintenance, not just ErrorInPrepareForMai…
DaanHoogland Oct 21, 2026
6036002
emit maintenance prepare error event only when entering ErrorInPrepar…
DaanHoogland Nov 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions api/src/main/java/com/cloud/event/EventTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,7 @@ public class EventTypes {
public static final String EVENT_MAINTENANCE_CANCEL = "MAINT.CANCEL";
public static final String EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE = "MAINT.CANCEL.PS";
public static final String EVENT_MAINTENANCE_PREPARE = "MAINT.PREPARE";
public static final String EVENT_MAINTENANCE_PREPARE_ERROR = "MAINT.PREPARE.ERROR";
Comment thread
DaanHoogland marked this conversation as resolved.
Resolved
public static final String EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE = "MAINT.PREPARE.PS";

// Primary storage pool
Expand Down Expand Up @@ -1046,6 +1047,7 @@ public class EventTypes {
entityEventDetails.put(EVENT_MAINTENANCE_CANCEL, Host.class);
entityEventDetails.put(EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE, Host.class);
entityEventDetails.put(EVENT_MAINTENANCE_PREPARE, Host.class);
entityEventDetails.put(EVENT_MAINTENANCE_PREPARE_ERROR, Host.class);
entityEventDetails.put(EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE, Host.class);

// Primary storage pool
Expand Down
15 changes: 15 additions & 0 deletions server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1723,16 +1723,31 @@ protected boolean setHostIntoErrorInMaintenance(HostVO host, List<VMInstanceVO>
_haMgr.cancelScheduledMigrations(host);
configureVncAccessForKVMHostFailedMigrations(host, errorVms);
resourceStateTransitTo(host, ResourceState.Event.UnableToMaintain, _nodeId);
emitMaintenanceErrorEvent(host, String.format("failed to put host %s into maintenance due to running VMs that could not be migrated or VMs in error state", host));
return false;
}

/**
* Set host into ErrorInPrepareForMaintenance state. The agent monitor calls this on every check while the errors
* persist, so the error event is only emitted when the host enters this state.
*/
protected boolean setHostIntoErrorInPrepareForMaintenance(HostVO host, List<VMInstanceVO> errorVms) throws NoTransitionException {
logger.debug("Host {} entering in PrepareForMaintenanceWithErrors state", host);
final boolean alreadyInErrorState = host.getResourceState() == ResourceState.ErrorInPrepareForMaintenance;
configureVncAccessForKVMHostFailedMigrations(host, errorVms);
resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId);
if (!alreadyInErrorState) {
emitMaintenanceErrorEvent(host, String.format("failed to prepare host %s for maintenance due to migration or VM state errors", host));
}
return false;
}

private void emitMaintenanceErrorEvent(HostVO host, String description) {
final CallContext context = CallContext.current();
ActionEventUtils.onCompletedActionEvent(context.getCallingUserId(), context.getCallingAccountId(),
EventVO.LEVEL_ERROR, EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR, description, host.getId(), null, 0);
}

protected boolean setHostIntoPrepareForMaintenanceAfterErrorsFixed(HostVO host) throws NoTransitionException {
logger.debug("Host {} entering in PrepareForMaintenance state as any previous corrections have been fixed", host);
resourceStateTransitTo(host, ResourceState.Event.ErrorsCorrected, _nodeId);
Expand Down
31 changes: 31 additions & 0 deletions server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import com.cloud.agent.api.GetVncPortCommand;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.ha.HighAvailabilityManager;
import com.cloud.host.Host;
Expand Down Expand Up @@ -79,6 +81,7 @@
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
Expand Down Expand Up @@ -521,27 +524,55 @@ private void setupErrorVms() {
}

private void verifyErrorInMaintenanceCalls() throws NoTransitionException {
when(host.getResourceState()).thenReturn(ResourceState.PrepareForMaintenance);
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).attemptMaintain(host);
verify(resourceManager).setHostIntoErrorInMaintenance(eq(host), any());
verify(resourceManager, never()).setHostIntoMaintenance(any());
verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMaintain), anyLong());
actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent(
anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR),
anyString(), eq(hostId), isNull(), eq(0L)));
Assert.assertFalse(enterMaintenanceMode);
}

private void verifyErrorInPrepareForMaintenanceCalls() throws NoTransitionException {
when(host.getResourceState()).thenReturn(ResourceState.PrepareForMaintenance);
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).attemptMaintain(host);
verify(resourceManager).setHostIntoErrorInPrepareForMaintenance(eq(host), any());
verify(resourceManager, never()).setHostIntoMaintenance(any());
verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong());
String expectedDescription = String.format("failed to prepare host %s for maintenance due to migration or VM state errors", host);
actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent(
anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR),
eq(expectedDescription), eq(hostId), isNull(), eq(0L)));
Comment thread
DaanHoogland marked this conversation as resolved.
Resolved
Assert.assertFalse(enterMaintenanceMode);
}

@Test
public void testCheckAndMaintainStillInErrorInPrepareForMaintenanceDoesNotRepeatErrorEvent() throws NoTransitionException {
setupFailedMigrations();
setupPendingMigrationRetries();
when(host.getResourceState()).thenReturn(ResourceState.ErrorInPrepareForMaintenance);
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).setHostIntoErrorInPrepareForMaintenance(eq(host), any());
verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong());
actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent(
anyLong(), anyLong(), anyString(), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR),
anyString(), anyLong(), any(), anyLong()), never());
Assert.assertFalse(enterMaintenanceMode);
}

@Test
public void testMaintenancePrepareErrorEventMapsToHostEntity() {
Assert.assertEquals(Host.class, EventTypes.getEntityClassForEvent(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR));
}

private void verifyReturnToPrepareForMaintenanceCalls() throws NoTransitionException {
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
verify(resourceManager).attemptMaintain(host);
Expand Down
Loading