fix HA-enabled VMs stuck in Running after out-of-band stop (OOM-kill) (#14)

* power state

* update

Author: poddm

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -209,8 +209,10 @@
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.exception.StorageAccessException;
 import com.cloud.exception.StorageUnavailableException;
+import com.cloud.ha.HaWorkVO;
 import com.cloud.ha.HighAvailabilityManager;
 import com.cloud.ha.HighAvailabilityManager.WorkType;
+import com.cloud.ha.dao.HighAvailabilityDao;
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
 import com.cloud.host.Status;
@@ -361,6 +363,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     @Inject
     private HighAvailabilityManager _haMgr;
     @Inject
+    private HighAvailabilityDao _haDao;
+    @Inject
     private HostPodDao _podDao;
     @Inject
     private DataCenterDao _dcDao;
@@ -5395,9 +5399,43 @@ private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) {
                     && HaVmRestartHostUp.value()
                     && vm.getHypervisorType() != HypervisorType.VMware
                     && vm.getHypervisorType() != HypervisorType.Hyperv) {
-                logger.info("Detected out-of-band stop of a HA enabled VM {}, will schedule restart.", vm);
+                logger.info("Detected out-of-band stop of HA enabled VM {}, transitioning to Stopped and scheduling HA restart (investigate=false).", vm);
                 if (!_haMgr.hasPendingHaWork(vm.getId())) {
-                    _haMgr.scheduleRestart(vm, true);
+                    // The power state report already confirmed the VM is off
+                    // (e.g. OOM-killed QEMU process). We cannot use _haMgr.scheduleRestart() because it
+                    // calls advanceStop(), which submits a VM work job and blocks waiting for completion.
+                    // This code runs on the AgentManager-Handler thread, and the VM job queue does not
+                    // dispatch jobs from this context — causing an indefinite block that leaves the VM
+                    // stuck in Running. Instead we release resources, transition to Stopped, and insert
+                    // an HA work item directly into op_ha_work (Step.Scheduled) for the HA worker to pick up.
+                    final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
+                    releaseVmResources(profile, true);
+
+                    // Save lastHostId before transition (stateTransitTo sets hostId=null)
+                    final Long lastHostId = vm.getHostId();
+                    try {
+                        stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, null);
+                    } catch (final NoTransitionException e) {
+                        logger.warn("Failed to transition VM {} to Stopped state: {}", vm, e.getMessage());
+                        return;
+                    }
+
+                    _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(),
+                            VM_SYNC_ALERT_SUBJECT, String.format("VM %s(%s) stopped out-of-band (OOM-killed or crashed). HA restart scheduled.",
+                                    vm.getHostName(), vm.getInstanceName()));
+
+                    // Insert HA work item directly — bypasses advanceStop/job queue entirely
+                    final VMInstanceVO refreshedVm = _vmDao.findByUuid(vm.getUuid());
+                    final Long haHostId = lastHostId != null ? lastHostId : refreshedVm.getLastHostId();
+                    if (haHostId == null || haHostId == 0L) {
+                        logger.warn("Cannot schedule HA for VM {} — no valid host_id available (would violate FK constraint).", vm);
+                    } else {
+                        final HaWorkVO work = new HaWorkVO(refreshedVm.getId(), refreshedVm.getType(),
+                                WorkType.HA, HighAvailabilityManager.Step.Scheduled, haHostId,
+                                refreshedVm.getState(), 0, refreshedVm.getUpdated(), null);
+                        _haDao.persist(work);
+                        logger.info("Scheduled VM for HA: {}", refreshedVm);
+                    }
                 } else {
                     logger.info("VM {} already has a pending HA task working on it.", vm);
                 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -533,6 +533,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_BLOCKED, PowerState.PowerOn);
         POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_NOSTATE, PowerState.PowerUnknown);
         POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_SHUTDOWN, PowerState.PowerOff);
+        // Report crashed domains as PowerOff immediately instead of omitting them
+        // from the report (which delays detection via the "missing VM" threshold).
+        POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_CRASHED, PowerState.PowerOff);
     }
 
     public VirtualRoutingResource virtRouterResource;