/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ha;
import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
import org.apache.hadoop.ha.TestNodeFencer.AlwaysSucceedFencer;
import org.apache.hadoop.ha.TestNodeFencer.AlwaysFailFencer;
import static org.apache.hadoop.ha.TestNodeFencer.setupFencer;
import org.apache.hadoop.security.AccessControlException;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.stubbing.answers.ThrowsException;
import static org.junit.Assert.*;
public class TestFailoverController {
private InetSocketAddress svc1Addr = new InetSocketAddress("svc1", 1234);
private InetSocketAddress svc2Addr = new InetSocketAddress("svc2", 5678);
private Configuration conf = new Configuration();
HAServiceStatus STATE_NOT_READY = new HAServiceStatus(HAServiceState.STANDBY)
.setNotReadyToBecomeActive("injected not ready");
@Test
public void testFailoverAndFailback() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
AlwaysSucceedFencer.fenceCalled = 0;
doFailover(svc1, svc2, false, false);
assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
AlwaysSucceedFencer.fenceCalled = 0;
doFailover(svc2, svc1, false, false);
assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFailoverFromStandbyToStandby() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.STANDBY, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
doFailover(svc1, svc2, false, false);
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverFromActiveToActive() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
fail("Can't failover to an already active service");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverWithoutPermission() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new AccessControlException("Access denied"))
.when(svc1.proxy).getServiceStatus();
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new AccessControlException("Access denied"))
.when(svc2.proxy).getServiceStatus();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
fail("Can't failover when access is denied");
} catch (FailoverFailedException ffe) {
assertTrue(ffe.getCause().getMessage().contains("Access denied"));
}
}
@Test
public void testFailoverToUnreadyService() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy)
.getServiceStatus();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
fail("Can't failover to a service that's not ready");
} catch (FailoverFailedException ffe) {
// Expected
if (!ffe.getMessage().contains("injected not ready")) {
throw ffe;
}
}
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
// Forcing it means we ignore readyToBecomeActive
doFailover(svc1, svc2, false, true);
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new HealthCheckFailedException("Failed!"))
.when(svc2.proxy).monitorHealth();
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
fail("Failover to unhealthy service");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFailoverFromFaultyServiceSucceeds() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc1.proxy).transitionToStandby(anyReqInfo());
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
AlwaysSucceedFencer.fenceCalled = 0;
try {
doFailover(svc1, svc2, false, false);
} catch (FailoverFailedException ffe) {
fail("Faulty active prevented failover");
}
// svc1 still thinks it's active, that's OK, it was fenced
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertSame(svc1, AlwaysSucceedFencer.fencedSvc);
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverFromFaultyServiceFencingFailure() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc1.proxy).transitionToStandby(anyReqInfo());
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
AlwaysFailFencer.fenceCalled = 0;
try {
doFailover(svc1, svc2, false, false);
fail("Failed over even though fencing failed");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(1, AlwaysFailFencer.fenceCalled);
assertSame(svc1, AlwaysFailFencer.fencedSvc);
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFencingFailureDuringFailover() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
AlwaysFailFencer.fenceCalled = 0;
try {
doFailover(svc1, svc2, true, false);
fail("Failed over even though fencing requested and failed");
} catch (FailoverFailedException ffe) {
// Expected
}
// If fencing was requested and it failed we don't try to make
// svc2 active anyway, and we don't failback to svc1.
assertEquals(1, AlwaysFailFencer.fenceCalled);
assertSame(svc1, AlwaysFailFencer.fencedSvc);
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testFailoverFromNonExistantServiceWithFencer() throws Exception {
DummyHAService svc1 = spy(new DummyHAService(null, svc1Addr));
// Getting a proxy to a dead server will throw IOException on call,
// not on creation of the proxy.
HAServiceProtocol errorThrowingProxy = Mockito.mock(HAServiceProtocol.class,
Mockito.withSettings()
.defaultAnswer(new ThrowsException(
new IOException("Could not connect to host")))
.extraInterfaces(Closeable.class));
Mockito.doNothing().when((Closeable)errorThrowingProxy).close();
Mockito.doReturn(errorThrowingProxy).when(svc1).getProxy(
Mockito.<Configuration>any(),
Mockito.anyInt());
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
} catch (FailoverFailedException ffe) {
fail("Non-existant active prevented failover");
}
// Verify that the proxy created to try to make it go to standby
// gracefully used the right rpc timeout
Mockito.verify(svc1).getProxy(
Mockito.<Configuration>any(),
Mockito.eq(
CommonConfigurationKeys.HA_FC_GRACEFUL_FENCE_TIMEOUT_DEFAULT));
// Don't check svc1 because we can't reach it, but that's OK, it's been fenced.
assertEquals(HAServiceState.ACTIVE, svc2.state);
}
@Test
public void testFailoverToNonExistantServiceFails() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = spy(new DummyHAService(null, svc2Addr));
Mockito.doThrow(new IOException("Failed to connect"))
.when(svc2).getProxy(Mockito.<Configuration>any(),
Mockito.anyInt());
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
fail("Failed over to a non-existant standby");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.ACTIVE, svc1.state);
}
@Test
public void testFailoverToFaultyServiceFailsbackOK() throws Exception {
DummyHAService svc1 = spy(new DummyHAService(HAServiceState.ACTIVE, svc1Addr));
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive(anyReqInfo());
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
fail("Failover to already active service");
} catch (FailoverFailedException ffe) {
// Expected
}
// svc1 went standby then back to active
verify(svc1.proxy).transitionToStandby(anyReqInfo());
verify(svc1.proxy).transitionToActive(anyReqInfo());
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testWeDontFailbackIfActiveWasFenced() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive(anyReqInfo());
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, true, false);
fail("Failed over to service that won't transition to active");
} catch (FailoverFailedException ffe) {
// Expected
}
// We failed to failover and did not failback because we fenced
// svc1 (we forced it), therefore svc1 and svc2 should be standby.
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testWeFenceOnFailbackIfTransitionToActiveFails() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive(anyReqInfo());
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
AlwaysSucceedFencer.fenceCalled = 0;
try {
doFailover(svc1, svc2, false, false);
fail("Failed over to service that won't transition to active");
} catch (FailoverFailedException ffe) {
// Expected
}
// We failed to failover. We did not fence svc1 because it cooperated
// and we didn't force it, so we failed back to svc1 and fenced svc2.
// Note svc2 still thinks it's active, that's OK, we fenced it.
assertEquals(HAServiceState.ACTIVE, svc1.state);
assertEquals(1, AlwaysSucceedFencer.fenceCalled);
assertSame(svc2, AlwaysSucceedFencer.fencedSvc);
}
private StateChangeRequestInfo anyReqInfo() {
return Mockito.<StateChangeRequestInfo>any();
}
@Test
public void testFailureToFenceOnFailbackFailsTheFailback() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new IOException("Failed!"))
.when(svc2.proxy).transitionToActive(anyReqInfo());
svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
AlwaysFailFencer.fenceCalled = 0;
try {
doFailover(svc1, svc2, false, false);
fail("Failed over to service that won't transition to active");
} catch (FailoverFailedException ffe) {
// Expected
}
// We did not fence svc1 because it cooperated and we didn't force it,
// we failed to failover so we fenced svc2, we failed to fence svc2
// so we did not failback to svc1, ie it's still standby.
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(1, AlwaysFailFencer.fenceCalled);
assertSame(svc2, AlwaysFailFencer.fencedSvc);
}
@Test
public void testFailbackToFaultyServiceFails() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc1.proxy).transitionToActive(anyReqInfo());
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
Mockito.doThrow(new ServiceFailedException("Failed!"))
.when(svc2.proxy).transitionToActive(anyReqInfo());
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
try {
doFailover(svc1, svc2, false, false);
fail("Failover to already active service");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(HAServiceState.STANDBY, svc1.state);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
@Test
public void testSelfFailoverFails() throws Exception {
DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
AlwaysSucceedFencer.fenceCalled = 0;
try {
doFailover(svc1, svc1, false, false);
fail("Can't failover to yourself");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
assertEquals(HAServiceState.ACTIVE, svc1.state);
try {
doFailover(svc2, svc2, false, false);
fail("Can't failover to yourself");
} catch (FailoverFailedException ffe) {
// Expected
}
assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
assertEquals(HAServiceState.STANDBY, svc2.state);
}
private void doFailover(HAServiceTarget tgt1, HAServiceTarget tgt2,
boolean forceFence, boolean forceActive) throws FailoverFailedException {
FailoverController fc = new FailoverController(conf,
RequestSource.REQUEST_BY_USER);
fc.failover(tgt1, tgt2, forceFence, forceActive);
}
}