TestFailoverWatcher.java

package com.xiaomi.infra.chronos.zookeeper;

import java.io.IOException;
import java.util.List;
import java.util.Properties;

import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

import static org.junit.Assert.*;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.zookeeper.KeeperException;

import com.xiaomi.infra.chronos.zookeeper.FailoverServer;
import com.xiaomi.infra.chronos.zookeeper.FailoverWatcher;
import com.xiaomi.infra.chronos.zookeeper.HostPort;
import com.xiaomi.infra.chronos.zookeeper.ZooKeeperUtil;

/**
 * Test {@link FailoverWatcher}
 */
public class TestFailoverWatcher {
  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(new Configuration());

  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    TEST_UTIL.startMiniZKCluster(1);
  }

  @AfterClass
  public static void tearDownAfterClass() throws Exception {
    TEST_UTIL.shutdownMiniZKCluster();
  }

  @Before
  public void resetZooKeeper() throws IOException, KeeperException {
    FailoverWatcher failoverWatcher = createFailoverWatcher(new HostPort("127.0.0.1", 10086));
    ZooKeeperUtil.deleteNodeRecursively(failoverWatcher, failoverWatcher.getBaseZnode());
    failoverWatcher.close();
  }

  private FailoverWatcher createFailoverWatcher(HostPort hostPort) throws IOException {
    Properties properties = new Properties();
    properties.setProperty(FailoverServer.SERVER_HOST, hostPort.getHost());
    properties.setProperty(FailoverServer.SERVER_PORT, String.valueOf(hostPort.getPort()));
    properties.setProperty(FailoverServer.BASE_ZNODE, "/test-failover");
    properties.setProperty(FailoverServer.ZK_QUORUM,
      ZKConfig.getZKQuorumServersString(TEST_UTIL.getConfiguration()));
    properties.setProperty(FailoverServer.SESSION_TIMEOUT, String.valueOf(3000));
    properties.setProperty(FailoverServer.CONNECT_RETRY_TIMES, String.valueOf(10));

    return new FailoverWatcher(properties);
  }

  @Test
  public void testInitZnode() throws IOException, KeeperException {
    FailoverWatcher failoverWatcher = createFailoverWatcher(new HostPort("127.0.0.1", 10086));

    assertTrue(ZooKeeperUtil.watchAndCheckExists(failoverWatcher, failoverWatcher.getBaseZnode()));
    assertTrue(ZooKeeperUtil.watchAndCheckExists(failoverWatcher,
      failoverWatcher.getBackupServersZnode()));

    failoverWatcher.close();
  }

  @Test
  public void testBlockUntilActive() throws IOException, KeeperException, InterruptedException {
    // the first server, expect to be active master
    HostPort hostPort1 = new HostPort("127.0.0.1", 11111);
    FailoverWatcher failoverWatcher1 = createFailoverWatcher(hostPort1);
    failoverWatcher1.blockUntilActive();

    assertTrue(failoverWatcher1.hasActiveServer());
    String activeServer = new String(ZooKeeperUtil.getDataAndWatch(failoverWatcher1,
      failoverWatcher1.masterZnode));
    assertTrue(activeServer.equals(hostPort1.getHostPort()));

    // the second server, expect to be backup master
    HostPort hostPort2 = new HostPort("127.0.0.1", 22222);
    final FailoverWatcher failoverWatcher2 = createFailoverWatcher(hostPort2);
    Thread thread2 = new Thread() {
      @Override
      public void run() {
        failoverWatcher2.blockUntilActive();
      }
    };
    thread2.start();

    Thread.sleep(500); // wait for second watcher to create znode
    List<String> backupMastersList = ZooKeeperUtil.listChildrenAndWatchForNewChildren(
      failoverWatcher1, failoverWatcher1.backupServersZnode);
    assertTrue(failoverWatcher2.hasActiveServer());
    assertTrue(backupMastersList.contains(failoverWatcher2.getHostPort().getHostPort()));

    thread2.interrupt();
    failoverWatcher1.close();
    failoverWatcher2.close();
  }

  @Test
  public void testRestartActiveMaster() throws IOException, KeeperException {
    // the first server, expect to be active master
    HostPort hostPort1 = new HostPort("127.0.0.1", 11111);
    FailoverWatcher failoverWatcher1 = createFailoverWatcher(hostPort1);
    failoverWatcher1.blockUntilActive();

    assertTrue(failoverWatcher1.hasActiveServer());
    String activeServer = new String(ZooKeeperUtil.getDataAndWatch(failoverWatcher1,
      failoverWatcher1.masterZnode));
    assertTrue(activeServer.equals(hostPort1.getHostPort()));

    // the same server start, expect to restart active master
    FailoverWatcher failoverWatcher2 = createFailoverWatcher(hostPort1);
    failoverWatcher2.blockUntilActive();
    assertTrue(failoverWatcher2.hasActiveServer());
    String activeServer2 = new String(ZooKeeperUtil.getDataAndWatch(failoverWatcher1,
      failoverWatcher1.masterZnode));
    assertTrue(activeServer2.equals(hostPort1.getHostPort()));

    failoverWatcher1.close();
    failoverWatcher2.close();
  }

  @Test
  public void testHandleMasterNodeChange() throws IOException, KeeperException,
      InterruptedException {
    HostPort hostPort1 = new HostPort("127.0.0.1", 11111);
    FailoverWatcher failoverWatcher1 = createFailoverWatcher(hostPort1);
    failoverWatcher1.blockUntilActive();

    // hostPort2 is backup master
    HostPort hostPort2 = new HostPort("127.0.0.1", 22222);
    final FailoverWatcher failoverWatcher2 = createFailoverWatcher(hostPort2);
    Thread thread2 = new Thread() {
      @Override
      public void run() {
        failoverWatcher2.blockUntilActive();
      }
    };
    thread2.start();

    // hostPort3 is backup master
    HostPort hostPort3 = new HostPort("127.0.0.1", 33333);
    final FailoverWatcher failoverWatcher3 = createFailoverWatcher(hostPort3);
    Thread thread3 = new Thread() {
      @Override
      public void run() {
        failoverWatcher3.blockUntilActive();
      }
    };
    thread3.start();

    // delete the master node, then one of them will become master, the other will be backup master
    ZooKeeperUtil.deleteNode(failoverWatcher1, failoverWatcher1.masterZnode);
    Thread.sleep(500); // wait for leader election
    String masterZnodeData = new String(ZooKeeperUtil.getDataAndWatch(failoverWatcher1,
      failoverWatcher1.masterZnode));
    String backupMasterNode = ZooKeeperUtil.listChildrenAndWatchForNewChildren(failoverWatcher1,
      failoverWatcher1.backupServersZnode).get(0);

    if (masterZnodeData.equals(hostPort2.getHostPort())) {
      if (!backupMasterNode.equals(hostPort3.getHostPort())) {
        assert false;
      }
    } else if (masterZnodeData.equals(hostPort3.getHostPort())) {
      if (!backupMasterNode.equals(hostPort2.getHostPort())) {
        assert false;
      }
    } else {
      assert false;
    }

    if (thread2.isAlive()) {
      thread2.interrupt();
    } else {
      thread3.interrupt();
    }

    failoverWatcher1.close();
    failoverWatcher2.close();
  }

  @Test
  public void testCloseMaster() throws KeeperException, IOException {
    HostPort hostPort1 = new HostPort("127.0.0.1", 11111);
    FailoverWatcher failoverWatcher1 = createFailoverWatcher(hostPort1);
    failoverWatcher1.blockUntilActive();

    HostPort hostPort2 = new HostPort("127.0.0.1", 22222);
    final FailoverWatcher failoverWatcher2 = createFailoverWatcher(hostPort2);
    Thread thread2 = new Thread() {
      @Override
      public void run() {
        failoverWatcher2.blockUntilActive();
      }
    };
    thread2.start();

    // delete master znode, then failoverWatcher2 will be master
    ZooKeeperUtil.deleteNode(failoverWatcher2, failoverWatcher2.getMasterZnode());
    String masterZnodeData = new String(ZooKeeperUtil.getDataAndWatch(failoverWatcher2,
      failoverWatcher2.masterZnode));
    List<String> backupMasterList = ZooKeeperUtil.listChildrenAndWatchForNewChildren(
      failoverWatcher2, failoverWatcher2.backupServersZnode);
    assertTrue(masterZnodeData.equals(failoverWatcher2.getHostPort().getHostPort()));
    assertTrue(backupMasterList.size() == 0);

    failoverWatcher1.close();
    failoverWatcher2.close();
  }

  @Test
  public void testCloseBackupMaster() throws IOException, KeeperException, InterruptedException {
    HostPort hostPort1 = new HostPort("127.0.0.1", 11111);
    FailoverWatcher failoverWatcher1 = createFailoverWatcher(hostPort1);
    failoverWatcher1.blockUntilActive();

    HostPort hostPort2 = new HostPort("127.0.0.1", 22222);
    final FailoverWatcher failoverWatcher2 = createFailoverWatcher(hostPort2);
    Thread thread2 = new Thread() {
      @Override
      public void run() {
        failoverWatcher2.blockUntilActive();
      }
    };
    thread2.start();

    Thread.sleep(500); // wait for backup master to init
    thread2.interrupt(); // close backup master and failoverWatcher1 is still active master
    ZooKeeperUtil.deleteNode(failoverWatcher1, failoverWatcher1.getBackupServersZnode() + "/"
        + hostPort2.getHostPort());

    String masterZnodeData = new String(ZooKeeperUtil.getDataAndWatch(failoverWatcher1,
      failoverWatcher1.masterZnode));
    List<String> backupMasterList = ZooKeeperUtil.listChildrenAndWatchForNewChildren(
      failoverWatcher1, failoverWatcher1.backupServersZnode);
    assertTrue(masterZnodeData.equals(failoverWatcher1.getHostPort().getHostPort()));
    assertTrue(backupMasterList.size() == 0);

    failoverWatcher1.close();
    failoverWatcher2.close();
  }

}