ClusterWithBackupFailoverTestBase.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.activemq.artemis.tests.integration.cluster.failover;

import java.util.HashSet;

import org.apache.activemq.artemis.api.core.client.ClientSession;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.core.server.cluster.impl.MessageLoadBalancingType;
import org.apache.activemq.artemis.tests.integration.IntegrationTestLogger;
import org.apache.activemq.artemis.tests.integration.cluster.distribution.ClusterTestBase;
import org.apache.activemq.artemis.tests.integration.cluster.util.SameProcessActiveMQServer;
import org.apache.activemq.artemis.tests.integration.cluster.util.TestableServer;
import org.junit.Before;
import org.junit.Test;

public abstract class ClusterWithBackupFailoverTestBase extends ClusterTestBase {

   protected static final String QUEUE_NAME = "queue0";
   protected static final String QUEUES_TESTADDRESS = "queues.testaddress";
   private static final IntegrationTestLogger log = IntegrationTestLogger.LOGGER;

   protected abstract void setupCluster(final MessageLoadBalancingType messageLoadBalancingType) throws Exception;

   protected abstract void setupServers() throws Exception;

   @Override
   @Before
   public void setUp() throws Exception {
      super.setUp();
      setupServers();
   }

   protected boolean isNetty() {
      return false;
   }

   @Test
   public void testFailLiveNodes() throws Throwable {
      setupCluster();

      startServers(3, 4, 5, 0, 1, 2);
      //startServers(0, 1, 2, 3, 4, 5);

      for (int i = 0; i < 3; i++) {
         waitForTopology(servers[i], 3, 3);
      }

      waitForFailoverTopology(3, 0, 1, 2);
      waitForFailoverTopology(4, 0, 1, 2);
      waitForFailoverTopology(5, 0, 1, 2);

      setupSessionFactory(0, 3, isNetty(), false);
      setupSessionFactory(1, 4, isNetty(), false);
      setupSessionFactory(2, 5, isNetty(), false);

      createQueue(0, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);
      createQueue(1, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);
      createQueue(2, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);

      addConsumer(0, 0, QUEUE_NAME, null);
      waitForBindings(0, QUEUES_TESTADDRESS, 1, 1, true);
      addConsumer(1, 1, QUEUE_NAME, null);
      waitForBindings(1, QUEUES_TESTADDRESS, 1, 1, true);
      addConsumer(2, 2, QUEUE_NAME, null);
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);

      waitForBindings();

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);
      Thread.sleep(1000);
      log.info("######### Topology on client = " + locators[0].getTopology().describe() + " locator = " + locators[0]);
      log.info("######### Crashing it........., sfs[0] = " + sfs[0]);
      failNode(0);

      waitForFailoverTopology(4, 3, 1, 2);
      waitForFailoverTopology(5, 3, 1, 2);

      // live nodes
      waitForBindings(1, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);
      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 1, 1, true);

      // live nodes
      waitForBindings(1, QUEUES_TESTADDRESS, 2, 2, false);
      waitForBindings(2, QUEUES_TESTADDRESS, 2, 2, false);
      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 2, 2, false);

      ClusterWithBackupFailoverTestBase.log.info("** now sending");

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      failNode(1);

      waitForFailoverTopology(5, 3, 4, 2);

      Thread.sleep(1000);
      // live nodes
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);
      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(4, QUEUES_TESTADDRESS, 1, 1, true);

      // live nodes
      waitForBindings(2, QUEUES_TESTADDRESS, 2, 2, false);
      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 2, 2, false);
      waitForBindings(4, QUEUES_TESTADDRESS, 2, 2, false);

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      failNode(2);

      Thread.sleep(1000);
      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(4, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(5, QUEUES_TESTADDRESS, 1, 1, true);

      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 2, 2, false);
      waitForBindings(4, QUEUES_TESTADDRESS, 2, 2, false);
      waitForBindings(5, QUEUES_TESTADDRESS, 2, 2, false);

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      removeConsumer(0);
      removeConsumer(1);
      removeConsumer(2);
   }

   private void waitForBindings() throws Exception {
      waitForBindings(0, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(1, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);

      waitForBindings(0, QUEUES_TESTADDRESS, 2, 2, false);
      waitForBindings(1, QUEUES_TESTADDRESS, 2, 2, false);
      waitForBindings(2, QUEUES_TESTADDRESS, 2, 2, false);
   }

   @Test
   public void testFailBackupNodes() throws Exception {
      setupCluster();

      startServers(3, 4, 5, 0, 1, 2);

      for (int i = 0; i < 3; i++) {
         waitForTopology(servers[i], 3, 3);
      }

      setupSessionFactory(0, 3, isNetty(), false);
      setupSessionFactory(1, 4, isNetty(), false);
      setupSessionFactory(2, 5, isNetty(), false);

      createQueue(0, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);
      createQueue(1, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);
      createQueue(2, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);

      addConsumer(0, 0, QUEUE_NAME, null);
      addConsumer(1, 1, QUEUE_NAME, null);
      addConsumer(2, 2, QUEUE_NAME, null);

      waitForBindings();

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      failNode(3);

      waitForBindings();

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      failNode(4);

      waitForBindings();

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      failNode(5);

      waitForBindings();

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      removeConsumer(0);
      removeConsumer(1);
      removeConsumer(2);
   }

   protected void setupCluster() throws Exception {
      setupCluster(MessageLoadBalancingType.ON_DEMAND);
   }

   protected void failNode(final int node) throws Exception {
      failNode(node, node);
   }

   /**
    * @param node             The node which we should fail
    * @param originalLiveNode The number of the original node, to locate session to fail
    * @throws Exception
    */
   protected void failNode(final int node, final int originalLiveNode) throws Exception {
      ClusterWithBackupFailoverTestBase.log.info("*** failing node " + node);

      ActiveMQServer server = getServer(node);

      TestableServer tstServer = new SameProcessActiveMQServer(server);

      ClientSession[] sessionsArray = exploreSessions(originalLiveNode);

      tstServer.crash(sessionsArray);
   }

   private ClientSession[] exploreSessions(final int node) {
      HashSet<ClientSession> sessions = new HashSet<>();

      for (ConsumerHolder holder : consumers) {
         if (holder != null && holder.getNode() == node && holder.getSession() != null) {
            sessions.add(holder.getSession());
         }
      }

      ClientSession[] sessionsArray = sessions.toArray(new ClientSession[sessions.size()]);
      return sessionsArray;
   }

   @Test
   public void testFailAllNodes() throws Exception {
      setupCluster();

      startServers(0, 1, 2, 3, 4, 5);

      setupSessionFactory(0, 3, isNetty(), false);
      setupSessionFactory(1, 4, isNetty(), false);
      setupSessionFactory(2, 5, isNetty(), false);

      waitForFailoverTopology(3, 0, 1, 2);
      waitForFailoverTopology(4, 0, 1, 2);
      waitForFailoverTopology(5, 0, 1, 2);

      createQueue(0, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);
      createQueue(1, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);
      createQueue(2, QUEUES_TESTADDRESS, QUEUE_NAME, null, true);

      addConsumer(0, 0, QUEUE_NAME, null);
      addConsumer(1, 1, QUEUE_NAME, null);
      addConsumer(2, 2, QUEUE_NAME, null);

      waitForBindings();

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      failNode(0);

      waitForFailoverTopology(4, 3, 1, 2);
      waitForFailoverTopology(5, 3, 1, 2);
      // live nodes
      waitForBindings(1, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);
      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 1, 1, true);

      // live nodes
      waitForBindings(1, QUEUES_TESTADDRESS, 2, 2, false);
      waitForBindings(2, QUEUES_TESTADDRESS, 2, 2, false);
      // activated backup nodes
      waitForBindings(3, QUEUES_TESTADDRESS, 2, 2, false);

      ClusterWithBackupFailoverTestBase.log.info("** now sending");

      send(0, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2);

      removeConsumer(0);
      failNode(3);

      // live nodes
      waitForBindings(1, QUEUES_TESTADDRESS, 1, 1, true);
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);

      // live nodes
      waitForBindings(1, QUEUES_TESTADDRESS, 1, 1, false);
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, false);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2);

      failNode(1);

      waitForFailoverTopology(5, 2, 4);
      // live nodes
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);
      // activated backup nodes
      waitForBindings(4, QUEUES_TESTADDRESS, 1, 1, true);

      // live nodes
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, false);
      // activated backup nodes
      waitForBindings(4, QUEUES_TESTADDRESS, 1, 1, false);

      send(1, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2);

      removeConsumer(1);

      // live nodes
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 1, true);
      // live nodes
      waitForBindings(2, QUEUES_TESTADDRESS, 1, 0, false);

      failNode(4, 1);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 2);

      failNode(2);

      // live nodes
      waitForBindings(5, QUEUES_TESTADDRESS, 1, 1, true);
      // live nodes
      waitForBindings(5, QUEUES_TESTADDRESS, 0, 0, false);

      send(2, QUEUES_TESTADDRESS, 10, false, null);
      verifyReceiveRoundRobinInSomeOrder(true, 10, 2);

      removeConsumer(2);
      failNode(5);
   }
}