/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.transport.failover;
import javax.jms.Connection;
import javax.jms.JMSException;
import javax.jms.MessageConsumer;
import javax.jms.Queue;
import javax.jms.Session;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.activemq.ActiveMQConnection;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.artemis.core.config.Configuration;
import org.apache.activemq.artemis.jms.server.config.impl.JMSConfigurationImpl;
import org.apache.activemq.artemis.jms.server.embedded.EmbeddedJMS;
import org.apache.activemq.broker.artemiswrapper.OpenwireArtemisBaseTest;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
/**
* Complex cluster test that will exercise the dynamic failover capabilities of
* a network of brokers. Using a networking of 3 brokers where the 3rd broker is
* removed and then added back in it is expected in each test that the number of
* connections on the client should start with 3, then have two after the 3rd
* broker is removed and then show 3 after the 3rd broker is reintroduced.
*/
public class FailoverComplexClusterTest extends OpenwireArtemisBaseTest {
private static final String BROKER_A_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61616";
private static final String BROKER_B_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61617";
private String clientUrl;
private EmbeddedJMS[] servers = new EmbeddedJMS[3];
private static final int NUMBER_OF_CLIENTS = 30;
private final List<ActiveMQConnection> connections = new ArrayList<>();
@Before
public void setUp() throws Exception {
}
//default setup for most tests
private void commonSetup() throws Exception {
Map<String, String> params = new HashMap<>();
params.put("rebalanceClusterClients", "true");
params.put("updateClusterClients", "true");
params.put("updateClusterClientsOnRemove", "true");
Configuration config0 = createConfig("localhost", 0, params);
Configuration config1 = createConfig("localhost", 1, params);
Configuration config2 = createConfig("localhost", 2, params);
deployClusterConfiguration(config0, 1, 2);
deployClusterConfiguration(config1, 0, 2);
deployClusterConfiguration(config2, 0, 1);
servers[0] = new EmbeddedJMS().setConfiguration(config0).setJmsConfiguration(new JMSConfigurationImpl());
servers[1] = new EmbeddedJMS().setConfiguration(config1).setJmsConfiguration(new JMSConfigurationImpl());
servers[2] = new EmbeddedJMS().setConfiguration(config2).setJmsConfiguration(new JMSConfigurationImpl());
servers[0].start();
servers[1].start();
servers[2].start();
Assert.assertTrue(servers[0].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, 3));
Assert.assertTrue(servers[1].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, 3));
Assert.assertTrue(servers[2].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, 3));
}
@After
public void tearDown() throws Exception {
shutdownClients();
for (EmbeddedJMS server : servers) {
if (server != null) {
server.stop();
}
}
}
/**
* Basic dynamic failover 3 broker test
*
* @throws Exception
*/
@Test
public void testThreeBrokerClusterSingleConnectorBasic() throws Exception {
commonSetup();
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
createClients();
Thread.sleep(3000);
runTests(false, null, null, null);
}
/**
* Tests a 3 broker configuration to ensure that the backup is random and
* supported in a cluster. useExponentialBackOff is set to false and
* maxReconnectAttempts is set to 1 to move through the list quickly for
* this test.
*
* @throws Exception
*/
@Test
public void testThreeBrokerClusterSingleConnectorBackupFailoverConfig() throws Exception {
commonSetup();
Thread.sleep(2000);
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")?backup=true&backupPoolSize=2&useExponentialBackOff=false&initialReconnectDelay=500");
createClients();
Thread.sleep(2000);
runTests(false, null, null, null);
}
/**
* Tests a 3 broker cluster that passes in connection params on the
* transport connector. Prior versions of AMQ passed the TC connection
* params to the client and this should not happen. The chosen param is not
* compatible with the client and will throw an error if used.
*
* @throws Exception
*/
@Test
public void testThreeBrokerClusterSingleConnectorWithParams() throws Exception {
commonSetup();
Thread.sleep(2000);
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
createClients();
Thread.sleep(2000);
runTests(false, null, null, null);
}
/**
* Tests a 3 broker cluster using a cluster filter of *
*
* @throws Exception
*/
@Test
public void testThreeBrokerClusterWithClusterFilter() throws Exception {
commonSetup();
Thread.sleep(2000);
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
createClients();
runTests(false, null, "*", null);
}
/**
* Test to verify that a broker with multiple transport connections only the
* one marked to update clients is propagate
*
* @throws Exception
*/
@Test
public void testThreeBrokerClusterMultipleConnectorBasic() throws Exception {
commonSetup();
Thread.sleep(2000);
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
createClients();
Thread.sleep(2000);
runTests(true, null, null, null);
}
/**
* Test to verify the reintroduction of the A Broker
*
* @throws Exception
*/
@Test
public void testOriginalBrokerRestart() throws Exception {
commonSetup();
Thread.sleep(2000);
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
createClients();
Thread.sleep(2000);
assertClientsConnectedToThreeBrokers();
stopServer(0);
Thread.sleep(5000);
assertClientsConnectedToTwoBrokers();
restartServer(0);
Thread.sleep(5000);
assertClientsConnectedToThreeBrokers();
}
/**
* Test to ensure clients are evenly to all available brokers in the
* network.
*
* @throws Exception
*/
@Test
public void testThreeBrokerClusterClientDistributions() throws Exception {
commonSetup();
Thread.sleep(2000);
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false&initialReconnectDelay=500");
createClients(100);
Thread.sleep(5000);
runClientDistributionTests(false, null, null, null);
}
/**
* Test to verify that clients are distributed with no less than 20% of the
* clients on any one broker.
*
* @throws Exception
*/
@Test
public void testThreeBrokerClusterDestinationFilter() throws Exception {
commonSetup();
Thread.sleep(2000);
setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
createClients();
runTests(false, null, null, "Queue.TEST.FOO.>");
}
@Test
public void testFailOverWithUpdateClientsOnRemove() throws Exception {
// Broker A
Configuration config0 = createConfig(0, "?rebalanceClusterClients=true&updateClusterClients=true&updateClusterClientsOnRemove=true");
// Broker B
Configuration config1 = createConfig(1, "?rebalanceClusterClients=true&updateClusterClients=true&updateClusterClientsOnRemove=true");
deployClusterConfiguration(config0, 1);
deployClusterConfiguration(config1, 0);
servers[0] = new EmbeddedJMS().setConfiguration(config0).setJmsConfiguration(new JMSConfigurationImpl());
servers[0].start();
servers[1] = new EmbeddedJMS().setConfiguration(config1).setJmsConfiguration(new JMSConfigurationImpl());
servers[1].start();
servers[0].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, 2);
servers[1].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, 2);
Thread.sleep(1000);
// create client connecting only to A. It should receive broker B address whet it connects to A.
setClientUrl("failover:(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=true");
createClients(1);
Thread.sleep(5000);
// We stop broker A.
servers[0].stop();
servers[1].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, 1);
Thread.sleep(5000);
// Client should failover to B.
assertAllConnectedTo(BROKER_B_CLIENT_TC_ADDRESS);
}
/**
* Runs a 3 Broker dynamic failover test: <br/>
* <ul>
* <li>asserts clients are distributed across all 3 brokers</li>
* <li>asserts clients are distributed across 2 brokers after removing the 3rd</li>
* <li>asserts clients are distributed across all 3 brokers after
* reintroducing the 3rd broker</li>
* </ul>
*
* @param multi
* @param tcParams
* @param clusterFilter
* @param destinationFilter
* @throws Exception
* @throws InterruptedException
*/
private void runTests(boolean multi,
String tcParams,
String clusterFilter,
String destinationFilter) throws Exception, InterruptedException {
assertClientsConnectedToThreeBrokers();
stopServer(2);
Thread.sleep(5000);
assertClientsConnectedToTwoBrokers();
restartServer(2);
Thread.sleep(5000);
assertClientsConnectedToThreeBrokers();
}
public void setClientUrl(String clientUrl) {
this.clientUrl = clientUrl;
}
protected void createClients() throws Exception {
createClients(NUMBER_OF_CLIENTS);
}
protected void createClients(int numOfClients) throws Exception {
ActiveMQConnectionFactory factory = new ActiveMQConnectionFactory(clientUrl);
for (int i = 0; i < numOfClients; i++) {
ActiveMQConnection c = (ActiveMQConnection) factory.createConnection();
c.start();
Session s = c.createSession(false, Session.AUTO_ACKNOWLEDGE);
Queue queue = s.createQueue(getClass().getName());
MessageConsumer consumer = s.createConsumer(queue);
connections.add(c);
}
}
protected void shutdownClients() throws JMSException {
for (Connection c : connections) {
c.close();
}
}
protected void assertClientsConnectedToThreeBrokers() {
Set<String> set = new HashSet<>();
for (ActiveMQConnection c : connections) {
if (c.getTransportChannel().getRemoteAddress() != null) {
set.add(c.getTransportChannel().getRemoteAddress());
}
}
Assert.assertTrue("Only 3 connections should be found: " + set, set.size() == 3);
}
protected void assertClientsConnectedToTwoBrokers() {
Set<String> set = new HashSet<>();
for (ActiveMQConnection c : connections) {
if (c.getTransportChannel().getRemoteAddress() != null) {
set.add(c.getTransportChannel().getRemoteAddress());
}
}
Assert.assertTrue("Only 2 connections should be found: " + set, set.size() == 2);
}
private void stopServer(int serverID) throws Exception {
servers[serverID].stop();
for (int i = 0; i < servers.length; i++) {
if (i != serverID) {
Assert.assertTrue(servers[i].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, servers.length - 1));
}
}
}
private void restartServer(int serverID) throws Exception {
servers[serverID].start();
for (int i = 0; i < servers.length; i++) {
Assert.assertTrue(servers[i].waitClusterForming(100, TimeUnit.MILLISECONDS, 20, servers.length));
}
}
private void runClientDistributionTests(boolean multi,
String tcParams,
String clusterFilter,
String destinationFilter) throws Exception, InterruptedException {
assertClientsConnectedToThreeBrokers();
//if 2/3 or more of total connections connect to one node, we consider it wrong
//if 1/4 or less of total connects to one node, we consider it wrong
assertClientsConnectionsEvenlyDistributed(.25, .67);
stopServer(2);
Thread.sleep(5000);
assertClientsConnectedToTwoBrokers();
//now there are only 2 nodes
//if 2/3 or more of total connections go to either node, we consider it wrong
//if 1/3 or less of total connections go to either node, we consider it wrong
assertClientsConnectionsEvenlyDistributed(.34, .67);
restartServer(2);
Thread.sleep(5000);
assertClientsConnectedToThreeBrokers();
//now back to 3 nodes. We assume at least the new node will
//have 1/10 of the total connections, and any node's connections
//won't exceed 50%
assertClientsConnectionsEvenlyDistributed(.10, .50);
}
protected void assertClientsConnectionsEvenlyDistributed(double minimumPercentage, double maximumPercentage) {
Map<String, Double> clientConnectionCounts = new HashMap<>();
int total = 0;
for (ActiveMQConnection c : connections) {
String key = c.getTransportChannel().getRemoteAddress();
if (key != null) {
total++;
if (clientConnectionCounts.containsKey(key)) {
double count = clientConnectionCounts.get(key);
count += 1.0;
clientConnectionCounts.put(key, count);
} else {
clientConnectionCounts.put(key, 1.0);
}
}
}
Set<String> keys = clientConnectionCounts.keySet();
List<String> errorMsgs = new ArrayList<>();
for (String key : keys) {
double count = clientConnectionCounts.get(key);
double percentage = count / total;
if (percentage < minimumPercentage || percentage > maximumPercentage) {
errorMsgs.add("Connections distribution expected to be within range [ " + minimumPercentage + ", " + maximumPercentage + "]. Actuall distribution was " + percentage + " for connection " + key);
}
if (errorMsgs.size() > 0) {
for (String err : errorMsgs) {
System.err.println(err);
}
Assert.fail("Test failed. Please see the log message for details");
}
}
}
protected void assertAllConnectedTo(String url) throws Exception {
for (ActiveMQConnection c : connections) {
Assert.assertEquals(url, c.getTransportChannel().getRemoteAddress());
}
}
}