/*
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.geode.internal.cache.tier.sockets;
import static org.junit.Assert.*;
import org.apache.geode.test.junit.categories.ClientSubscriptionTest;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.apache.geode.test.dunit.Assert;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.NetworkUtils;
import org.apache.geode.test.dunit.Wait;
import org.apache.geode.test.dunit.WaitCriterion;
import org.apache.geode.test.junit.categories.DistributedTest;
@Category({DistributedTest.class, ClientSubscriptionTest.class})
public class RedundancyLevelPart2DUnitTest extends RedundancyLevelTestBase {
@BeforeClass
public static void caseSetUp() throws Exception {
disconnectAllFromDS();
}
private void waitConnectedServers(final int expected) {
WaitCriterion wc = new WaitCriterion() {
public boolean done() {
return expected == pool.getConnectedServerCount();
}
public String description() {
return "Connected server count (" + pool.getConnectedServerCount() + ") never became "
+ expected;
}
};
Wait.waitForCriterion(wc, 2 * 60 * 1000, 1000, true);
}
/**
* Redundancy level specified & less than total Eps. If an EP dies & is part of the fail over list
* , then it should be removed from live server map & added to dead server map. A new EP should be
* picked from the live server map to compensate for the failure. The EP failed also happened to
* be a Primary EP so a new EP from fail over set should become the primary and make sure that CCP
* is created on the server with relevant interest registartion. Failure Detection by LSM
*/
@Test
public void testRedundancySpecifiedPrimaryEPFails() {
try {
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
1);
waitConnectedServers(4);
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
// assertIndexDetailsEquals(0, proxy.getDeadServers().size());
verifyOrderOfEndpoints();
server0.invoke(() -> RedundancyLevelTestBase.stopServer());
// pause(5000);
verifyDeadServers(1);
verifyRedundantServersContain(SERVER3);
verifyLiveAndRedundantServers(3, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// assertTrue(pool.getRedundantNames()
// .contains(SERVER2));
assertTrue(pool.getPrimaryName().equals(SERVER2));
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
// assertIndexDetailsEquals(1, proxy.getDeadServers().size());
} catch (Exception ex) {
Assert.fail("test failed due to exception in test testRedundancySpecifiedPrimaryEPFails ",
ex);
}
}
/**
* Redundancy level specified & less than total Eps. If an EP dies & is part of the fail over list
* , then it should be removed from live server map & added to dead server map. A new EP should be
* picked from the live server map to compensate for the failure. The EP failed also happened to
* be a Primary EP so a new EP from fail over set should become the primary and make sure that CCP
* is created on the server with relevant interest registartion. Failure Detection by CCU
*/
@Test
public void testRedundancySpecifiedPrimaryEPFailsDetectionByCCU() {
try {
FailOverDetectionByCCU = true;
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
1, 3000, 100);
waitConnectedServers(4);
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
// assertIndexDetailsEquals(0, proxy.getDeadServers().size());
verifyOrderOfEndpoints();
server0.invoke(() -> RedundancyLevelTestBase.stopServer());
// pause(5000);
verifyDeadServers(1);
verifyRedundantServersContain(SERVER3);
verifyLiveAndRedundantServers(3, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// assertTrue(pool.getRedundantNames()
// .contains(SERVER2));
assertTrue(pool.getPrimaryName().equals(SERVER2));
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
// assertIndexDetailsEquals(1, proxy.getDeadServers().size());
} catch (Exception ex) {
Assert.fail(
"test failed due to exception in test testRedundancySpecifiedPrimaryEPFailsDetectionByCCU ",
ex);
}
}
/*
* Redundancy level specified & less than total Eps. If an EP dies & is part of the fail over list
* , then it should be removed from live server map & added to dead server map. A new EP should be
* picked from the live server map to compensate for the failure. The EP failed also happened to
* be a Primary EP so a new EP from fail over set should become the primary and make sure that CCP
* is created on the server with relevant interest registartion. Failure Detection by Register
* Interest
*/
@Test
public void testRedundancySpecifiedPrimaryEPFailsDetectionByRegisterInterest() {
try {
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
1, 3000, 100);
waitConnectedServers(4);
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
// assertIndexDetailsEquals(0, proxy.getDeadServers().size());
verifyOrderOfEndpoints();
server0.invoke(() -> RedundancyLevelTestBase.stopServer());
// pause(5000);
createEntriesK1andK2();
registerK1AndK2();
verifyDeadServers(1);
verifyRedundantServersContain(SERVER3);
verifyLiveAndRedundantServers(3, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// assertTrue(pool.getRedundantNames()
// .contains(SERVER2));
assertTrue(pool.getPrimaryName().equals(SERVER2));
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
// assertIndexDetailsEquals(1, proxy.getDeadServers().size());
} catch (Exception ex) {
Assert.fail(
"test failed due to exception in test testRedundancySpecifiedPrimaryEPFailsDetectionByRegisterInterest ",
ex);
}
}
/*
* Redundancy level specified & less than total Eps. If an EP dies & is part of the fail over list
* , then it should be removed from live server map & added to dead server map. A new EP should be
* picked from the live server map to compensate for the failure. The EP failed also happened to
* be a Primary EP so a new EP from fail over set should become the primary and make sure that CCP
* is created on the server with relevant interest registartion. Failure Detection by Unregister
* Interest
*/
@Test
public void testRedundancySpecifiedPrimaryEPFailsDetectionByUnregisterInterest() {
try {
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
1, 3000, 100);
waitConnectedServers(4);
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
// assertIndexDetailsEquals(0, proxy.getDeadServers().size());
verifyOrderOfEndpoints();
server0.invoke(() -> RedundancyLevelTestBase.stopServer());
// pause(5000);
unregisterInterest();
verifyDeadServers(1);
verifyRedundantServersContain(SERVER3);
verifyLiveAndRedundantServers(3, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// assertTrue(pool.getRedundantNames()
// .contains(SERVER2));
assertTrue(pool.getPrimaryName().equals(SERVER2));
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
// assertIndexDetailsEquals(1, proxy.getDeadServers().size());
} catch (Exception ex) {
Assert.fail(
"test failed due to exception in test testRedundancySpecifiedPrimaryEPFailsDetectionByUnregisterInterest ",
ex);
}
}
/*
* Redundancy level specified & less than total Eps. If an EP dies & is part of the fail over list
* , then it should be removed from live server map & added to dead server map. A new EP should be
* picked from the live server map to compensate for the failure. The EP failed also happened to
* be a Primary EP so a new EP from fail over set should become the primary and make sure that CCP
* is created on the server with relevant interest registartion. Failure Detection by put
* operation
*/
@Test
public void testRedundancySpecifiedPrimaryEPFailsDetectionByPut() {
try {
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
1, 3000, 100);
waitConnectedServers(4);
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
// assertIndexDetailsEquals(0, proxy.getDeadServers().size());
verifyOrderOfEndpoints();
server0.invoke(() -> RedundancyLevelTestBase.stopServer());
// pause(5000);
doPuts();
verifyDeadServers(1);
verifyRedundantServersContain(SERVER3);
verifyLiveAndRedundantServers(3, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// assertTrue(pool.getRedundantNames()
// .contains(SERVER2));
assertTrue(pool.getPrimaryName().equals(SERVER2));
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
// assertIndexDetailsEquals(1, proxy.getDeadServers().size());
} catch (Exception ex) {
Assert.fail(
"test failed due to exception in test testRedundancySpecifiedPrimaryEPFailsDetectionByPut ",
ex);
}
}
/*
* If there are 4 servers in Live Server Map with redundancy level as 1. Kill the primary &
* secondary. Two Eps should be added from the live server & the existing EP in the set should be
* the new primary and make sure that CCP is created on both the server with relevant interest
* registartion.
*/
@Test
public void testRedundancySpecifiedPrimarySecondaryEPFails() {
try {
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
1);
waitConnectedServers(4);
assertEquals(1, pool.getRedundantNames().size());
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
assertFalse(pool.getRedundantNames().contains(SERVER3));
assertFalse(pool.getRedundantNames().contains(SERVER4));
verifyOrderOfEndpoints();
server0.invoke(() -> RedundancyLevelTestBase.stopServer());
server1.invoke(() -> RedundancyLevelTestBase.stopServer());
// pause(5000);
verifyLiveAndRedundantServers(2, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// Not Sure
// assertTrue(pool.getPrimaryName().equals(SERVER2));
server2.invoke(() -> RedundancyLevelTestBase.verifyInterestRegistration());
// assertTrue(pool.getRedundantNames().contains(SERVER3));
server3.invoke(() -> RedundancyLevelTestBase.verifyInterestRegistration());
} catch (Exception ex) {
Assert.fail(
"test failed due to exception in test testRedundancySpecifiedPrimarySecondaryEPFails ",
ex);
}
}
/**
* There are 4 Eps in Live serevr Map with redundancy level as 2. Kill two Eps (excluding the
* primary). As a result live server map will contain 2 , active list will contain two & dead
* server map will contain 2. Redundancy is unsatisfied by 1. Bring up one EP . The DSM should add
* the alive EP to the active end point & live server map with primary unchnaged. Also make sure
* that CCP is created on the server with relevant interest registartion. Bringing the 4th EP
* alive should simply add it to Live server map.
*/
@Test
public void testRedundancySpecifiedEPFails() {
try {
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
2);
waitConnectedServers(4);
assertEquals(2, pool.getRedundantNames().size());
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
assertTrue(pool.getRedundantNames().contains(SERVER3));
assertFalse(pool.getRedundantNames().contains(SERVER4));
// kill non primary EPs
verifyOrderOfEndpoints();
server1.invoke(() -> RedundancyLevelTestBase.stopServer());
server2.invoke(() -> RedundancyLevelTestBase.stopServer());
verifyDeadServers(2);
// assertIndexDetailsEquals(2, pool.getConnectedServerCount());
// assertIndexDetailsEquals(2, proxy.getDeadServers().size());
// pause(10000);
verifyLiveAndRedundantServers(2, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// bring up one server.
server1.invoke(() -> RedundancyLevelTestBase.startServer());
// pause(10000);
verifyLiveAndRedundantServers(3, 2);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(3, pool.getRedundantNames().size());
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
assertTrue(pool.getPrimaryName().equals(SERVER1));
verifyRedundantServersContain(SERVER2);
verifyRedundantServersContain(SERVER4);
server1.invoke(() -> RedundancyLevelTestBase.verifyInterestRegistration());
// bring up another server should get added to live server map only and
// not to the active server as redundancy level is satisfied.
server2.invoke(() -> RedundancyLevelTestBase.startServer());
// pause(10000);
Wait.pause(1000);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(3, pool.getRedundantNames().size());
// assertIndexDetailsEquals(4, pool.getConnectedServerCount());
server2.invoke(() -> RedundancyLevelTestBase.verifyNoCCP());
} catch (Exception ex) {
Assert.fail("test failed due to exception in test testRedundancySpecifiedEPFails ", ex);
}
}
/**
* Redundancy level specified but not satisfied, new EP is added then it should be added in Live
* server map as well as failover set and make sure that CCP is created on the server with
* relevant interest registartion.
*/
@Test
public void testRedundancyLevelSpecifiedButNotSatisfied() {
try {
// stop two secondaries
server2.invoke(() -> RedundancyLevelTestBase.stopServer());
server1.invoke(() -> RedundancyLevelTestBase.stopServer());
// make sure that the client connects to only two servers and
// redundancyLevel
// unsatisfied with one
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
2);
// let the client connect to servers
// pause(10000);
verifyLiveAndRedundantServers(2, 1);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(2, pool.getRedundantNames().size());
// assertIndexDetailsEquals(2, pool.getConnectedServerCount());
assertTrue(pool.getRedundantNames().contains(SERVER4));
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertFalse(pool.getRedundantNames().contains(SERVER3));
assertFalse(pool.getRedundantNames().contains(SERVER2));
// start server
server2.invoke(() -> RedundancyLevelTestBase.startServer());
// pause(10000);
verifyLiveAndRedundantServers(3, 2);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
// assertIndexDetailsEquals(3, pool.getRedundantNames().size());
assertTrue(pool.getRedundantNames().contains(SERVER4));
assertTrue(pool.getRedundantNames().contains(SERVER3));
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertFalse(pool.getRedundantNames().contains(SERVER2));
server2.invoke(() -> RedundancyLevelTestBase.verifyInterestRegistration());
// verify that redundancy level is satisfied
server1.invoke(() -> RedundancyLevelTestBase.startServer());
// pause(10000);
Wait.pause(1000);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(3, pool.getRedundantNames().size());
// assertIndexDetailsEquals(4, pool.getConnectedServerCount());
assertTrue(pool.getRedundantNames().contains(SERVER4));
assertTrue(pool.getRedundantNames().contains(SERVER3));
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertFalse(pool.getRedundantNames().contains(SERVER2));
server1.invoke(() -> RedundancyLevelTestBase.verifyNoCCP());
} catch (Exception ex) {
Assert.fail("test failed due to exception in test noRedundancyLevelServerFail ", ex);
}
}
/**
* Redundancy level specified and satisfied, new EP is added then it should be added only in Live
* server map and make sure that no CCP is created on the server.
*/
@Test
public void testRedundancyLevelSpecifiedAndSatisfied() {
try {
// TODO: Yogesh
server1.invoke(() -> RedundancyLevelTestBase.stopServer());
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
2);
// let the client connect to servers
// pause(10000);
verifyLiveAndRedundantServers(3, 2);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(3, pool.getRedundantNames().size());
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
assertTrue(pool.getRedundantNames().contains(SERVER3));
assertTrue(pool.getRedundantNames().contains(SERVER4));
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertFalse(pool.getRedundantNames().contains(SERVER2));
// start server
server1.invoke(() -> RedundancyLevelTestBase.startServer());
Wait.pause(1000);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(3, pool.getRedundantNames().size());
// assertIndexDetailsEquals(4, pool.getConnectedServerCount());
assertTrue(pool.getRedundantNames().contains(SERVER3));
assertTrue(pool.getRedundantNames().contains(SERVER4));
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertFalse(pool.getRedundantNames().contains(SERVER2));
server1.invoke(() -> RedundancyLevelTestBase.verifyNoCCP());
} catch (Exception ex) {
Assert.fail("test failed due to exception in test noRedundancyLevelServerFail ", ex);
}
}
/**
* Redundancy level not specified, new EP is added then it should be added in live server map as
* well as failover set and make sure that CCP is created on the server with relevant interest
* registartion.
*/
@Test
public void testRedundancyLevelNotSpecified() {
try {
// TODO: Yogesh
server2.invoke(() -> RedundancyLevelTestBase.stopServer());
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
-1/* not specified */);
// let the client connect to servers
// pause(10000);
verifyLiveAndRedundantServers(3, 2);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(1, pool.getRedundantNames().size());
// assertIndexDetailsEquals(3, pool.getConnectedServerCount());
assertFalse(pool.getRedundantNames().contains(SERVER1));
assertFalse(pool.getRedundantNames().contains(SERVER3));
assertTrue(pool.getRedundantNames().contains(SERVER2));
assertTrue(pool.getPrimaryName().equals(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER4));
// start server
server2.invoke(() -> RedundancyLevelTestBase.startServer());
// pause(10000);
verifyLiveAndRedundantServers(4, 3);
verifyOrderOfEndpoints();
// assertIndexDetailsEquals(1, pool.getRedundantNames().size());
// assertIndexDetailsEquals(4, pool.getConnectedServerCount());
// assertTrue(pool.getRedundantNames()
// .contains(SERVER1));
assertTrue(pool.getRedundantNames().contains(SERVER2));
assertTrue(pool.getRedundantNames().contains(SERVER3));
assertTrue(pool.getRedundantNames().contains(SERVER4));
assertTrue(pool.getPrimaryName().equals(SERVER1));
server2.invoke(() -> RedundancyLevelTestBase.verifyInterestRegistration());
} catch (Exception ex) {
Assert.fail("test failed due to exception in test noRedundancyLevelServerFail ", ex);
}
}
/**
* Redundancy level specified is more than the total EndPoints. In such situation there should not
* be any exception & all the EPs should has CacheClientProxy created.
*/
@Test
public void testRedundancySpecifiedMoreThanEPs() {
try {
createClientCache(NetworkUtils.getServerHostName(Host.getHost(0)), PORT1, PORT2, PORT3, PORT4,
5);
assertEquals(3, pool.getRedundantNames().size());
server0.invoke(() -> RedundancyLevelTestBase.verifyCCP());
server1.invoke(() -> RedundancyLevelTestBase.verifyCCP());
server2.invoke(() -> RedundancyLevelTestBase.verifyCCP());
server3.invoke(() -> RedundancyLevelTestBase.verifyCCP());
} catch (Exception ex) {
ex.printStackTrace();
Assert.fail("test failed due to exception in test testRedundancySpecifiedMoreThanEPs ", ex);
}
}
}