From 5442d767353f3881805a152cf62bd38783925414 Mon Sep 17 00:00:00 2001 From: Nikolaus Eppinger Date: Fri, 10 Jul 2026 15:05:21 +0200 Subject: [PATCH] KVM: add storage-heartbeat fencing fallback when OOBM fence fails When a KVM host suffers a complete power failure, its BMC (IPMI/iDRAC/iLO) usually goes down with it. The Host HA framework's fence operation relies solely on out-of-band management, so fencing can never succeed in this scenario: the host remains stuck in the Fencing state indefinitely and scheduleRestartForVmsOnHost() is never invoked, leaving all HA-enabled VMs down until an operator intervenes - defeating the purpose of HA for the most common total-failure scenario. At that point CloudStack has already proven the host dead: health checks failed, activity checks found no VM disk activity, and the neighbouring hosts positively report the host's storage heartbeat as expired - the same evidence the legacy KVMFencer has always accepted as sufficient proof of fencing. This change adds an opt-in fallback (cluster setting kvm.ha.fence.on.storage.heartbeat, default false): when the OOBM fence operation fails, the host is considered fenced only if the neighbouring hosts positively report its storage heartbeat as expired (Status.Down from KVMHostActivityChecker). The host is never considered fenced when any check still sees it alive, when no neighbour can confirm, or when the heartbeat check itself fails - absence of evidence never counts as evidence of death. With the setting disabled (default) the behaviour is unchanged. --- .../apache/cloudstack/kvm/ha/KVMHAConfig.java | 7 ++ .../cloudstack/kvm/ha/KVMHAProvider.java | 42 +++++++++- .../cloudstack/kvm/ha/KVMHostHATest.java | 80 ++++++++++++++++++- 3 files changed, 126 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java index 3fbb5340fcce..456d88bbadb1 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java @@ -52,4 +52,11 @@ public interface KVMHAConfig { ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", "The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster); + + ConfigKey KvmHAFenceOnStorageHeartbeat = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat", "false", + "Whether to consider a host fenced when the out-of-band management fence operation fails (e.g. a complete power failure that takes down the BMC together with the host) " + + "but the neighbouring hosts in the cluster positively report the host's storage heartbeat as expired. The host is never considered fenced when any check still " + + "sees storage activity from it or when the heartbeat check itself fails. The heartbeat semantics follow the existing checks, including " + + "'kvm.ha.fence.host.if.heartbeat.fails.on.storage'. When disabled, fencing relies solely on out-of-band management.", + true, ConfigKey.Scope.Cluster); } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index f0b5cfc337de..533276bf8744 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -21,6 +21,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; +import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.api.response.OutOfBandManagementResponse; @@ -89,17 +90,53 @@ public boolean fence(Host r) throws HAFenceException { try { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null); - return resp.getSuccess(); + if (resp.getSuccess()) { + return true; + } + logger.warn("OOBM fence operation failed for the host {}", r); + return fenceHostViaStorageHeartbeat(r); } else { logger.warn("OOBM fence operation failed for this host {}", r); - return false; + return fenceHostViaStorageHeartbeat(r); } } catch (Exception e){ logger.warn("OOBM service is not configured or enabled for this host {} error is {}", r, e.getMessage()); + if (fenceHostViaStorageHeartbeat(r)) { + return true; + } throw new HAFenceException(String.format("OBM service is not configured or enabled for this host %s", r.getName()), e); } } + /** + * Fallback fencing for when out-of-band management cannot power off the host, e.g. a + * complete power failure that takes down the BMC (IPMI/iDRAC/iLO) together with the host. + * The host is considered fenced only when the neighbouring hosts in the cluster positively + * report its storage heartbeat as expired; it is never considered fenced when any check + * still sees it alive or when the heartbeat check itself fails. + */ + protected boolean fenceHostViaStorageHeartbeat(final Host host) { + if (!isStorageHeartbeatFencingEnabled(host)) { + return false; + } + try { + final Status hostStatus = hostActivityChecker.getHostAgentStatus(host); + if (hostStatus == Status.Down) { + logger.warn("Neighbouring hosts report the storage heartbeat of the host {} as expired, considering it fenced as [{}] is enabled", + host, KVMHAConfig.KvmHAFenceOnStorageHeartbeat.key()); + return true; + } + logger.warn("Not considering the host {} fenced via storage heartbeat as its reported status is [{}]", host, hostStatus); + } catch (Exception e) { + logger.warn("Storage heartbeat fencing check failed for the host {}", host, e); + } + return false; + } + + protected boolean isStorageHeartbeatFencingEnabled(final Host host) { + return KVMHAConfig.KvmHAFenceOnStorageHeartbeat.valueIn(host.getClusterId()); + } + @Override public HAResource.ResourceSubType resourceSubType() { return HAResource.ResourceSubType.KVM; @@ -152,6 +189,7 @@ public ConfigKey[] getConfigKeys() { KVMHAConfig.KvmHADegradedMaxPeriod, KVMHAConfig.KvmHARecoverWaitPeriod, KVMHAConfig.KvmHARecoverAttemptThreshold, + KVMHAConfig.KvmHAFenceOnStorageHeartbeat, }; } } diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java index a94fb01475a3..4c2dd8d2f8d1 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/kvm/ha/KVMHostHATest.java @@ -20,10 +20,18 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.apache.cloudstack.api.response.OutOfBandManagementResponse; import org.apache.cloudstack.ha.provider.HACheckerException; +import org.apache.cloudstack.ha.provider.HAFenceException; +import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerOperation; +import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService; import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; @@ -33,6 +41,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; +import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor.HypervisorType; @RunWith(MockitoJUnitRunner.class) @@ -42,12 +51,21 @@ public class KVMHostHATest { private Host host; @Mock private KVMHostActivityChecker kvmHostActivityChecker; + @Mock + private OutOfBandManagementService outOfBandManagementService; private KVMHAProvider kvmHAProvider; @Before public void setup() { - kvmHAProvider = new KVMHAProvider(); + kvmHAProvider = spy(new KVMHAProvider()); kvmHAProvider.hostActivityChecker = kvmHostActivityChecker; + kvmHAProvider.outOfBandManagementService = outOfBandManagementService; + } + + private OutOfBandManagementResponse oobmResponse(final boolean success) { + final OutOfBandManagementResponse response = new OutOfBandManagementResponse(); + response.setSuccess(success); + return response; } @Test @@ -80,4 +98,64 @@ public void testHostActivityForDownHost() throws HACheckerException, StorageUnav assertFalse(kvmHAProvider.hasActivity(host, dt)); } + @Test + public void testFenceSucceedsViaOobm() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(true)); + + assertTrue(kvmHAProvider.fence(host)); + + verify(kvmHostActivityChecker, never()).getHostAgentStatus(host); + } + + @Test + public void testFenceOobmFailedFallbackDisabled() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false)); + doReturn(false).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + + assertFalse(kvmHAProvider.fence(host)); + + verify(kvmHostActivityChecker, never()).getHostAgentStatus(host); + } + + @Test + public void testFenceOobmFailedStorageHeartbeatReportsHostDown() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false)); + doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Down); + + assertTrue(kvmHAProvider.fence(host)); + } + + @Test + public void testFenceOobmFailedStorageHeartbeatSeesHostAlive() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenReturn(oobmResponse(false)); + doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Disconnected); + + assertFalse(kvmHAProvider.fence(host)); + } + + @Test + public void testFenceOobmExceptionStorageHeartbeatReportsHostDown() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenThrow(new RuntimeException("BMC unreachable")); + doReturn(true).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + when(kvmHostActivityChecker.getHostAgentStatus(host)).thenReturn(Status.Down); + + assertTrue(kvmHAProvider.fence(host)); + } + + @Test(expected = HAFenceException.class) + public void testFenceOobmExceptionFallbackDisabledThrows() throws HAFenceException { + when(outOfBandManagementService.isOutOfBandManagementEnabled(host)).thenReturn(true); + when(outOfBandManagementService.executePowerOperation(host, PowerOperation.OFF, null)).thenThrow(new RuntimeException("BMC unreachable")); + doReturn(false).when(kvmHAProvider).isStorageHeartbeatFencingEnabled(host); + + kvmHAProvider.fence(host); + } + }