/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Consistency;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.RpcRetryingCallerImpl;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.replication.regionserver.TestRegionReplicaReplicationEndpoint;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
/**
* Tests failover of secondary region replicas.
*/
@RunWith(Parameterized.class)
@Category(LargeTests.class)
public class TestRegionReplicaFailover {
private static final Log LOG = LogFactory.getLog(TestRegionReplicaReplicationEndpoint.class);
static {
((Log4JLogger)RpcRetryingCallerImpl.LOG).getLogger().setLevel(Level.ALL);
}
private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
private static final int NB_SERVERS = 3;
protected final byte[][] families = new byte[][] {HBaseTestingUtility.fam1,
HBaseTestingUtility.fam2, HBaseTestingUtility.fam3};
protected final byte[] fam = HBaseTestingUtility.fam1;
protected final byte[] qual1 = Bytes.toBytes("qual1");
protected final byte[] value1 = Bytes.toBytes("value1");
protected final byte[] row = Bytes.toBytes("rowA");
protected final byte[] row2 = Bytes.toBytes("rowB");
@Rule public TestName name = new TestName();
private HTableDescriptor htd;
/*
* We are testing with dist log split and dist log replay separately
*/
@Parameters
public static Collection<Object[]> getParameters() {
Object[][] params =
new Boolean[][] { /*{true}, Disable DLR!!! It is going to be removed*/ {false} };
return Arrays.asList(params);
}
@Parameterized.Parameter(0)
public boolean distributedLogReplay;
@Before
public void before() throws Exception {
Configuration conf = HTU.getConfiguration();
// Up the handlers; this test needs more than usual.
conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true);
conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_WAIT_FOR_PRIMARY_FLUSH_CONF_KEY, true);
conf.setInt("replication.stats.thread.period.seconds", 5);
conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, distributedLogReplay);
HTU.startMiniCluster(NB_SERVERS);
htd = HTU.createTableDescriptor(
name.getMethodName().substring(0, name.getMethodName().length()-3));
htd.setRegionReplication(3);
HTU.getAdmin().createTable(htd);
}
@After
public void after() throws Exception {
HTU.deleteTableIfAny(htd.getTableName());
HTU.shutdownMiniCluster();
}
/**
* Tests the case where a newly created table with region replicas and no data, the secondary
* region replicas are available to read immediately.
*/
@Test(timeout = 60000)
public void testSecondaryRegionWithEmptyRegion() throws IOException {
// Create a new table with region replication, don't put any data. Test that the secondary
// region replica is available to read.
try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
Table table = connection.getTable(htd.getTableName())) {
Get get = new Get(row);
get.setConsistency(Consistency.TIMELINE);
get.setReplicaId(1);
table.get(get); // this should not block
}
}
/**
* Tests the case where if there is some data in the primary region, reopening the region replicas
* (enable/disable table, etc) makes the region replicas readable.
* @throws IOException
*/
@Test(timeout = 60000)
public void testSecondaryRegionWithNonEmptyRegion() throws IOException {
// Create a new table with region replication and load some data
// than disable and enable the table again and verify the data from secondary
try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
Table table = connection.getTable(htd.getTableName())) {
HTU.loadNumericRows(table, fam, 0, 1000);
HTU.getAdmin().disableTable(htd.getTableName());
HTU.getAdmin().enableTable(htd.getTableName());
HTU.verifyNumericRows(table, fam, 0, 1000, 1);
}
}
/**
* Tests the case where killing a primary region with unflushed data recovers
*/
@Test (timeout = 120000)
public void testPrimaryRegionKill() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
Table table = connection.getTable(htd.getTableName())) {
HTU.loadNumericRows(table, fam, 0, 1000);
// wal replication is async, we have to wait until the replication catches up, or we timeout
verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);
// we should not have flushed files now, but data in memstores of primary and secondary
// kill the primary region replica now, and ensure that when it comes back up, we can still
// read from it the same data from primary and secondaries
boolean aborted = false;
for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
for (Region r : rs.getRegionServer().getOnlineRegions(htd.getTableName())) {
if (r.getRegionInfo().getReplicaId() == 0) {
LOG.info("Aborting region server hosting primary region replica");
rs.getRegionServer().abort("for test");
aborted = true;
break;
}
}
}
assertTrue(aborted);
// wal replication is async, we have to wait until the replication catches up, or we timeout
verifyNumericRowsWithTimeout(table, fam, 0, 1000, 0, 30000);
verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);
}
// restart the region server
HTU.getMiniHBaseCluster().startRegionServer();
}
/** wal replication is async, we have to wait until the replication catches up, or we timeout
*/
private void verifyNumericRowsWithTimeout(final Table table, final byte[] f, final int startRow,
final int endRow, final int replicaId, final long timeout) throws Exception {
try {
HTU.waitFor(timeout, new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
try {
HTU.verifyNumericRows(table, f, startRow, endRow, replicaId);
return true;
} catch (AssertionError ae) {
return false;
}
}
});
} catch (Throwable t) {
// ignore this, but redo the verify do get the actual exception
HTU.verifyNumericRows(table, f, startRow, endRow, replicaId);
}
}
/**
* Tests the case where killing a secondary region with unflushed data recovers, and the replica
* becomes available to read again shortly.
*/
@Test (timeout = 120000)
public void testSecondaryRegionKill() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
Table table = connection.getTable(htd.getTableName())) {
HTU.loadNumericRows(table, fam, 0, 1000);
// wait for some time to ensure that async wal replication does it's magic
verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);
// we should not have flushed files now, but data in memstores of primary and secondary
// kill the secondary region replica now, and ensure that when it comes back up, we can still
// read from it the same data
boolean aborted = false;
for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
for (Region r : rs.getRegionServer().getOnlineRegions(htd.getTableName())) {
if (r.getRegionInfo().getReplicaId() == 1) {
LOG.info("Aborting region server hosting secondary region replica");
rs.getRegionServer().abort("for test");
aborted = true;
break;
}
}
}
assertTrue(aborted);
Threads.sleep(5000);
HTU.verifyNumericRows(table, fam, 0, 1000, 1);
HTU.verifyNumericRows(table, fam, 0, 1000, 2);
}
// restart the region server
HTU.getMiniHBaseCluster().startRegionServer();
}
/**
* Tests the case where there are 3 region replicas and the primary is continuously accepting
* new writes while one of the secondaries is killed. Verification is done for both of the
* secondary replicas.
*/
@Test (timeout = 120000)
public void testSecondaryRegionKillWhilePrimaryIsAcceptingWrites() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
Table table = connection.getTable(htd.getTableName());
Admin admin = connection.getAdmin()) {
// start a thread to do the loading of primary
HTU.loadNumericRows(table, fam, 0, 1000); // start with some base
admin.flush(table.getName());
HTU.loadNumericRows(table, fam, 1000, 2000);
final AtomicReference<Throwable> ex = new AtomicReference<>(null);
final AtomicBoolean done = new AtomicBoolean(false);
final AtomicInteger key = new AtomicInteger(2000);
Thread loader = new Thread() {
@Override
public void run() {
while (!done.get()) {
try {
HTU.loadNumericRows(table, fam, key.get(), key.get()+1000);
key.addAndGet(1000);
} catch (Throwable e) {
ex.compareAndSet(null, e);
}
}
}
};
loader.start();
Thread aborter = new Thread() {
@Override
public void run() {
try {
boolean aborted = false;
for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
for (Region r : rs.getRegionServer().getOnlineRegions(htd.getTableName())) {
if (r.getRegionInfo().getReplicaId() == 1) {
LOG.info("Aborting region server hosting secondary region replica");
rs.getRegionServer().abort("for test");
aborted = true;
}
}
}
assertTrue(aborted);
} catch (Throwable e) {
ex.compareAndSet(null, e);
}
};
};
aborter.start();
aborter.join();
done.set(true);
loader.join();
assertNull(ex.get());
assertTrue(key.get() > 1000); // assert that the test is working as designed
LOG.info("Loaded up to key :" + key.get());
verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 0, 30000);
verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 1, 30000);
verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 2, 30000);
}
// restart the region server
HTU.getMiniHBaseCluster().startRegionServer();
}
/**
* Tests the case where we are creating a table with a lot of regions and replicas. Opening region
* replicas should not block handlers on RS indefinitely.
*/
@Test (timeout = 120000)
public void testLotsOfRegionReplicas() throws IOException {
int numRegions = NB_SERVERS * 20;
int regionReplication = 10;
String tableName = htd.getTableName().getNameAsString() + "2";
htd = HTU.createTableDescriptor(tableName);
htd.setRegionReplication(regionReplication);
// dont care about splits themselves too much
byte[] startKey = Bytes.toBytes("aaa");
byte[] endKey = Bytes.toBytes("zzz");
byte[][] splits = HTU.getRegionSplitStartKeys(startKey, endKey, numRegions);
HTU.getAdmin().createTable(htd, startKey, endKey, numRegions);
try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
Table table = connection.getTable(htd.getTableName())) {
for (int i = 1; i < splits.length; i++) {
for (int j = 0; j < regionReplication; j++) {
Get get = new Get(splits[i]);
get.setConsistency(Consistency.TIMELINE);
get.setReplicaId(j);
table.get(get); // this should not block. Regions should be coming online
}
}
}
HTU.deleteTableIfAny(TableName.valueOf(tableName));
}
}