/*
* Copyright 2010 Outerthought bvba
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.lilyproject.hadooptestfw;
import javax.management.ObjectName;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.ReflectionUtils;
import org.lilyproject.hadooptestfw.fork.HBaseTestingUtility;
import org.lilyproject.util.io.Closer;
import org.lilyproject.util.jmx.JmxLiaison;
import org.lilyproject.util.test.TestHomeUtil;
/**
* Provides access to HBase, either by starting an embedded HBase or by connecting to a running HBase.
*
* <p>This is intended for usage in test cases.
*
* <p><b>VERY VERY IMPORTANT</b>: when connecting to an existing HBase, this class will DELETE ALL ROWS
* FROM ALL TABLES!
*/
public class HBaseProxy {
private Mode mode;
private Configuration conf;
private HBaseTestingUtility hbaseTestUtil;
private File testHome;
private CleanupUtil cleanupUtil;
private boolean cleanStateOnConnect = true;
private boolean enableMapReduce = false;
private boolean clearData = true;
private final Log log = LogFactory.getLog(getClass());
private ReplicationPeerUtil mbean = new ReplicationPeerUtil();
public enum Mode {EMBED, CONNECT}
public static String HBASE_MODE_PROP_NAME = "lily.hbaseproxy.mode";
public HBaseProxy() throws IOException {
this(null);
}
public HBaseProxy(Mode mode) throws IOException {
this(mode, true);
}
/**
* Creates new HBaseProxy
*
* @param mode either EMBED or CONNECT
* @param clearData if true, clears the data directories upon shutdown
* @throws IOException
*/
public HBaseProxy(Mode mode, boolean clearData) throws IOException {
this.clearData = clearData;
if (mode == null) {
String hbaseModeProp = System.getProperty(HBASE_MODE_PROP_NAME);
if (hbaseModeProp == null || hbaseModeProp.equals("") || hbaseModeProp.equals("embed")) {
this.mode = Mode.EMBED;
} else if (hbaseModeProp.equals("connect")) {
this.mode = Mode.CONNECT;
} else {
throw new RuntimeException("Unexpected value for " + HBASE_MODE_PROP_NAME + ": " + hbaseModeProp);
}
} else {
this.mode = mode;
}
}
public Mode getMode() {
return mode;
}
public void setTestHome(File testHome) throws IOException {
if (mode != Mode.EMBED) {
throw new RuntimeException("testHome should only be set when mode is EMBED");
}
this.testHome = testHome;
}
private void initTestHome() throws IOException {
if (testHome == null) {
testHome = TestHomeUtil.createTestHome("lily-hbaseproxy-");
}
FileUtils.forceMkdir(testHome);
}
public boolean getCleanStateOnConnect() {
return cleanStateOnConnect;
}
public void setCleanStateOnConnect(boolean cleanStateOnConnect) {
this.cleanStateOnConnect = cleanStateOnConnect;
}
public boolean getEnableMapReduce() {
return enableMapReduce;
}
public void setEnableMapReduce(boolean enableMapReduce) {
this.enableMapReduce = enableMapReduce;
}
public HBaseTestingUtility getHBaseTestingUtility() {
return hbaseTestUtil;
}
public void start() throws Exception {
start(Collections.<String, byte[]>emptyMap());
}
/**
* @param timestampReusingTables map containing table name as key and column family as value. Since HBase does
* not support supporting writing data older than a deletion thombstone, these tables
* will be compacted and waited for until inserting data works again.
*/
public void start(Map<String, byte[]> timestampReusingTables) throws Exception {
System.out.println("HBaseProxy mode: " + mode);
conf = HBaseConfiguration.create();
switch (mode) {
case EMBED:
addHBaseTestProps(conf);
addUserProps(conf);
initTestHome();
System.out.println("HBaseProxy embedded mode temp dir: " + testHome.getAbsolutePath());
hbaseTestUtil = HBaseTestingUtilityFactory.create(conf, testHome, clearData);
hbaseTestUtil.startMiniCluster(1);
if (enableMapReduce) {
hbaseTestUtil.startMiniMapReduceCluster(1);
}
writeConfiguration(testHome, conf);
// In the past, it happened that HMaster would not become initialized, blocking later on
// the proper shutdown of the mini cluster. Now added this as an early warning mechanism.
long before = System.currentTimeMillis();
while (!hbaseTestUtil.getMiniHBaseCluster().getMaster().isInitialized()) {
if (System.currentTimeMillis() - before > 60000) {
throw new RuntimeException("HMaster.isInitialized() does not become true.");
}
System.out.println("Waiting for HMaster to be initialized");
Thread.sleep(500);
}
conf = hbaseTestUtil.getConfiguration();
cleanupUtil = new CleanupUtil(conf, getZkConnectString());
break;
case CONNECT:
conf.set("hbase.zookeeper.quorum", "localhost");
conf.set("hbase.zookeeper.property.clientPort", "2181");
conf.set("hbase.replication", "true");
addUserProps(conf);
cleanupUtil = new CleanupUtil(conf, getZkConnectString());
if (cleanStateOnConnect) {
cleanupUtil.cleanZooKeeper();
Map<String, byte[]> allTimestampReusingTables = new HashMap<String, byte[]>();
allTimestampReusingTables.putAll(cleanupUtil.getDefaultTimestampReusingTables());
allTimestampReusingTables.putAll(timestampReusingTables);
cleanupUtil.cleanTables(allTimestampReusingTables);
List<String> removedPeers = cleanupUtil.cleanHBaseReplicas();
for (String removedPeer : removedPeers) {
waitOnReplicationPeerStopped(removedPeer);
}
}
break;
default:
throw new RuntimeException("Unexpected mode: " + mode);
}
ManagementFactory.getPlatformMBeanServer().registerMBean(mbean,
new ObjectName("LilyHBaseProxy:name=ReplicationPeer"));
}
/**
* Dumps the hadoop and hbase configuration. Useful as a reference if other applications want to use the
* same configuration to connect with the hadoop cluster.
*
* @param testHome directory in which to dump the configuration (it will create a conf subdir inside)
* @param conf the configuration
*/
private void writeConfiguration(File testHome, Configuration conf) throws IOException {
final File confDir = new File(testHome, "conf");
final boolean confDirCreated = confDir.mkdir();
if (!confDirCreated) {
throw new IOException("failed to create " + confDir);
}
// dumping everything into multiple xxx-site.xml files.. so that the expected files are definitely there
for (String filename : Arrays.asList("core-site.xml", "mapred-site.xml")) {
final BufferedOutputStream out =
new BufferedOutputStream(new FileOutputStream(new File(confDir, filename)));
try {
conf.writeXml(out);
} finally {
out.close();
}
}
}
public String getZkConnectString() {
return conf.get("hbase.zookeeper.quorum") + ":" + conf.get("hbase.zookeeper.property.clientPort");
}
/**
* Adds all system property prefixed with "lily.test.hbase." to the HBase configuration.
*/
private void addUserProps(Configuration conf) {
Properties sysProps = System.getProperties();
for (Map.Entry<Object, Object> entry : sysProps.entrySet()) {
String name = entry.getKey().toString();
if (name.startsWith("lily.test.hbase.")) {
String hbasePropName = name.substring("lily.test.".length());
conf.set(hbasePropName, entry.getValue().toString());
} else if (name.startsWith("lily.test.hbase-site.")) {
String hbasePropName = name.substring("lily.test.hbase-site.".length());
conf.set(hbasePropName, entry.getValue().toString());
}
}
}
protected static void addHBaseTestProps(Configuration conf) {
// The following properties are from HBase's src/test/resources/hbase-site.xml
conf.set("hbase.regionserver.msginterval", "1000");
conf.set("hbase.client.pause", "5000");
conf.set("hbase.client.retries.number", "4");
conf.set("hbase.master.meta.thread.rescanfrequency", "10000");
conf.set("hbase.server.thread.wakefrequency", "1000");
conf.set("hbase.regionserver.handler.count", "5");
conf.set("hbase.master.info.port", "-1");
conf.set("hbase.regionserver.info.port", "-1");
conf.set("hbase.regionserver.info.port.auto", "true");
conf.set("hbase.master.lease.thread.wakefrequency", "3000");
conf.set("hbase.regionserver.optionalcacheflushinterval", "1000");
conf.set("hbase.regionserver.safemode", "false");
}
public void stop() throws Exception {
if (mode == Mode.EMBED) {
// Since HBase mini cluster shutdown has a tendency of sometimes failing (hanging waiting on master
// to end), add a protection for this so that we do not run indefinitely. Especially important not to
// annoy the other projects on our Hudson server.
Thread stopHBaseThread = new Thread() {
@Override
public void run() {
try {
hbaseTestUtil.shutdownMiniCluster();
hbaseTestUtil = null;
} catch (Exception e) {
System.out.println("Error shutting down mini cluster.");
e.printStackTrace();
}
}
};
stopHBaseThread.start();
stopHBaseThread.join(60000);
if (stopHBaseThread.isAlive()) {
System.err.println("Unable to stop embedded mini cluster within predetermined timeout.");
System.err.println("Dumping stack for future investigation.");
ReflectionUtils.printThreadInfo(new PrintWriter(System.out), "Thread dump");
System.out.println(
"Will now try to interrupt the mini-cluster-stop-thread and give it some more time to end.");
stopHBaseThread.interrupt();
stopHBaseThread.join(20000);
throw new Exception("Failed to stop the mini cluster within the predetermined timeout.");
}
}
// Close connections with HBase and HBase's ZooKeeper handles
//HConnectionManager.deleteConnectionInfo(CONF, true);
HConnectionManager.deleteAllConnections(true);
// Close all HDFS connections
FileSystem.closeAll();
conf = null;
if (clearData && testHome != null) {
TestHomeUtil.cleanupTestHome(testHome);
}
ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName("LilyHBaseProxy:name=ReplicationPeer"));
}
public Configuration getConf() {
return conf;
}
public FileSystem getBlobFS() throws IOException, URISyntaxException {
if (mode == Mode.EMBED) {
return hbaseTestUtil.getDFSCluster().getFileSystem();
} else {
String dfsUri = System.getProperty("lily.test.dfs");
if (dfsUri == null) {
dfsUri = "hdfs://localhost:8020";
}
return FileSystem.get(new URI(dfsUri), getConf());
}
}
/**
* Cleans all data from the hbase tables.
*
* <p>Should only be called when lily-server is not running.
*/
public void cleanTables() throws Exception {
cleanupUtil.cleanTables();
}
/**
* Cleans all blobs from the hdfs blobstore
*
* <p>Should only be called when lily-server is not running.
*/
public void cleanBlobStore() throws Exception {
cleanupUtil.cleanBlobStore(getBlobFS().getUri());
}
public void rollHLog() throws Exception {
HBaseAdmin admin = new HBaseAdmin(conf);
try {
Collection<ServerName> serverNames = admin.getClusterStatus().getServers();
if (serverNames.size() != 1) {
throw new RuntimeException("Expected exactly one region server, but got: " + serverNames.size());
}
admin.rollHLogWriter(serverNames.iterator().next().getServerName());
} finally {
Closer.close(admin);
}
}
/**
* Wait for all outstanding waledit's that are currently in the hlog(s) to be replicated. Any new waledits
* produced during the calling of this method won't be processed.
*
* <p>To avoid any timing issues, after adding a replication peer you will want to call
* {@link #waitOnReplicationPeerReady(String)} to be sure the current logs are in the queue
* of the new peer and that the peer's mbean is registered, otherwise this method might skip
* that peer (usually will go so fast that this problem doesn't really exist, but just to be sure).</p>
*/
public boolean waitOnReplication(long timeout) throws Exception {
// Wait for the SEP to have processed all events.
// The idea is as follows:
// - we want to be sure hbase replication processed all outstanding events in the hlog
// - therefore, roll the current hlog
// - if the queue of hlogs to be processed by replication contains only the current hlog (the newly
// rolled one), all events in the previous hlog(s) will have been processed
//
// It only works for one region server (which is the case for the test framework) and of course
// assumes that new hlogs aren't being created in the meantime, but that is under all reasonable
// circumstances the case.
// It does ignore new edits that are happening after/during the call of this method, which is a good thing.
// Make sure there actually is something within the hlog, otherwise it won't roll
// This assumes the record table exits (doing the same with the .META. tables gives an exception
// "Failed openScanner" at KeyComparator.compareWithoutRow in connect mode)
HTable table = new HTable(conf, "-ROOT-");
Delete delete = new Delete(Bytes.toBytes("i-am-quite-sure-this-row-does-not-exist-ha-ha-ha"));
table.delete(delete);
// Roll the hlog
rollHLog();
// Force creation of a new HLog
delete = new Delete(Bytes.toBytes("i-am-quite-sure-this-row-does-not-exist-ha-ha-ha-2"));
table.delete(delete);
table.close();
// Using JMX, query the size of the queue of hlogs to be processed for each replication source
JmxLiaison jmxLiaison = new JmxLiaison();
jmxLiaison.connect(mode == Mode.EMBED);
ObjectName replicationSources = new ObjectName("hadoop:service=Replication,name=ReplicationSource for *");
Set<ObjectName> mbeans = jmxLiaison.queryNames(replicationSources);
long tryUntil = System.currentTimeMillis() + timeout;
nextMBean: for (ObjectName mbean : mbeans) {
int logQSize = Integer.MAX_VALUE;
while (logQSize > 0 && System.currentTimeMillis() < tryUntil) {
logQSize = (Integer)jmxLiaison.getAttribute(mbean, "sizeOfLogQueue");
// logQSize == 0 means there is one active hlog that is polled by replication
// and none that are queued for later processing
// System.out.println("hlog q size is " + logQSize + " for " + mbean.toString() + " max wait left is " +
// (tryUntil - System.currentTimeMillis()));
if (logQSize == 0) {
continue nextMBean;
} else {
Thread.sleep(100);
}
}
return false;
}
return true;
}
/**
* Iteratively wait until no more sep events were handled. Useful for testing sep processors that cause new sep
* events, which can cause even more sep events, and so on.
*
* See notes about {@link #waitOnReplicationPeerReady(String)} in the docs of {@link #waitOnReplication(long)}
*
* @param timeout
* @return
* @throws Exception
*/
public boolean waitOnSepIdle(long timeout) throws Exception {
JmxLiaison jmxLiaison = new JmxLiaison();
jmxLiaison.connect(mode == HBaseProxy.Mode.EMBED);
Map<String, Long> currentTimeStamp = getLastSepTimestamps(jmxLiaison);
Map<String, Long> lastTimeStamp = null;
int count = 0;
long tryUntil = System.currentTimeMillis() + timeout;
while(! currentTimeStamp.equals(lastTimeStamp)){
if (System.currentTimeMillis() > tryUntil) return false;
log.debug("waiting for sep to idle, iteration " + count++);
waitOnReplication(timeout);
lastTimeStamp = currentTimeStamp;
currentTimeStamp = getLastSepTimestamps(jmxLiaison);
}
return true;
}
/**
* obtains a map of the replication ids to the last sep event timestamp
*/
private Map<String, Long> getLastSepTimestamps(JmxLiaison jmxLiaison) throws Exception {
ObjectName replicationSources = new ObjectName("hadoop:service=SEP,name=*");
Set<ObjectName> mbeans = jmxLiaison.queryNames(replicationSources);
Map<String, Long> result = new HashMap<String, Long>(mbeans.size());
for (ObjectName mbean : mbeans) {
result.put(mbean.getKeyProperty("name"), (Long) jmxLiaison.getAttribute(mbean, "lastSepTimestamp"));
}
return result;
}
/**
* After adding a new replication peer, this waits for the replication source in the region server to be started.
*/
public void waitOnReplicationPeerReady(String peerId) throws Exception {
if (mode == Mode.EMBED) {
mbean.waitOnReplicationPeerReady(peerId);
} else {
JmxLiaison jmxLiaison = new JmxLiaison();
jmxLiaison.connect(false);
jmxLiaison.invoke(new ObjectName("LilyHBaseProxy:name=ReplicationPeer"), "waitOnReplicationPeerReady",
peerId);
jmxLiaison.disconnect();
}
}
/**
* After removing a replication peer, this waits for the replication source in the region server to be stopped,
* and will as well unregister its mbean (a workaround because this is missing in hbase at the time of this
* writing -- hbase 0.94.3)
*/
public void waitOnReplicationPeerStopped(String peerId) throws Exception {
if (mode == Mode.EMBED) {
mbean.waitOnReplicationPeerStopped(peerId);
} else {
JmxLiaison jmxLiaison = new JmxLiaison();
jmxLiaison.connect(false);
jmxLiaison.invoke(new ObjectName("LilyHBaseProxy:name=ReplicationPeer"), "waitOnReplicationPeerStopped",
peerId);
jmxLiaison.disconnect();
}
}
}