/*
* Copyright 2012 NGDATA nv
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.lilyproject.lilyservertestfw.launcher;
import javax.management.ObjectName;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.LogManager;
import com.google.common.collect.Lists;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.util.ReflectionUtils;
import org.lilyproject.cli.BaseCliTool;
import org.lilyproject.hadooptestfw.CleanupUtil;
import org.lilyproject.hadooptestfw.JavaLoggingToLog4jRedirector;
import org.lilyproject.hadooptestfw.ReplicationPeerUtil;
import org.lilyproject.lilyservertestfw.TemplateDir;
import org.lilyproject.solrtestfw.SolrProxy;
import org.lilyproject.util.Version;
import org.lilyproject.util.test.TestHomeUtil;
import org.slf4j.bridge.SLF4JBridgeHandler;
public class LilyLauncher extends BaseCliTool implements LilyLauncherMBean {
private Option enableHadoopOption;
private Option enableSolrOption;
private Option enableLilyOption;
private Option enableHbaseIndexerOption;
private Option dataDirOption;
private Option prepareOption;
private File testHome;
private boolean clearData = true;
private final HadoopLauncherService hadoopService = new HadoopLauncherService();
private final SolrLauncherService solrService = new SolrLauncherService();
private final LilyLauncherService lilyService = new LilyLauncherService();
private final HbaseIndexerLauncherService hbaseIndexerLauncherService = new HbaseIndexerLauncherService();
private final List<LauncherService> allServices = new ArrayList<LauncherService>();
private final List<LauncherService> enabledServices = new ArrayList<LauncherService>();
boolean enableHadoop;
boolean enableSolr;
boolean enableLily;
boolean enableHbaseIndexer;
private final Log log = LogFactory.getLog(getClass());
public LilyLauncher() {
allServices.add(hadoopService);
allServices.add(solrService);
allServices.add(lilyService);
allServices.add(hbaseIndexerLauncherService);
}
@Override
protected String getCmdName() {
return "launch-test-lily";
}
@Override
protected String getVersion() {
return Version.readVersion("org.lilyproject", "lily-server-test-fw");
}
@Override
@SuppressWarnings("static-access")
public List<Option> getOptions() {
List<Option> options = super.getOptions();
enableHadoopOption = OptionBuilder
.withDescription("Start the Hadoop services (ZK/DFS/MR/HBase)")
.withLongOpt("hadoop")
.create("hadoop");
options.add(enableHadoopOption);
enableSolrOption = OptionBuilder
.withDescription("Start the Solr service")
.withLongOpt("solr")
.create("solr");
options.add(enableSolrOption);
enableLilyOption = OptionBuilder
.withDescription("Start the Lily service")
.withLongOpt("lily")
.create("lily");
options.add(enableLilyOption);
enableHbaseIndexerOption = OptionBuilder
.withDescription("Start the HBase Indexer Service")
.withLongOpt("hbase-indexer")
.create("indexer");
dataDirOption = OptionBuilder
.withDescription("Directory where data should be stored, instead of a temporary directory. " +
"Can be used to restart from previous state.")
.hasArg()
.withArgName("path")
.withLongOpt("data-dir")
.create("d");
options.add(dataDirOption);
prepareOption = OptionBuilder
.withDescription("Create a template data directory, this will allow faster startup "
+ "in the future (when not using -d)")
.withLongOpt("prepare")
.create("p");
options.add(prepareOption);
for (LauncherService service : allServices) {
service.addOptions(options);
}
return options;
}
public static void main(String[] args) throws Exception {
new LilyLauncher().start(args);
}
@Override
public int run(CommandLine cmd) throws Exception {
int result = super.run(cmd);
if (result != 0) {
return result;
}
// Forward JDK logging to SLF4J
LogManager.getLogManager().reset();
LogManager.getLogManager().getLogger("").addHandler(new SLF4JBridgeHandler());
LogManager.getLogManager().getLogger("").setLevel(Level.ALL);
boolean prepareMode = cmd.hasOption(prepareOption.getOpt());
//
// Figure out what to start
//
enableHadoop = cmd.hasOption(enableHadoopOption.getOpt());
enableSolr = cmd.hasOption(enableSolrOption.getOpt());
enableLily = cmd.hasOption(enableLilyOption.getOpt());
enableHbaseIndexer = cmd.hasOption(enableHbaseIndexerOption.getOpt());
// When running prepare mode, or if none of the services are explicitly enabled,
// we default to starting them all. Otherwise we only start those that are enabled.
if (!enableHadoop && !enableSolr && !enableLily && !enableHbaseIndexer) {
enableHadoop = true;
enableSolr = true;
enableLily = true;
enableHbaseIndexer = true;
}
if (prepareMode) {
// (in prepare mode: always start everything)
System.out.println("-------------------------------------------------------------");
System.out.println("Running in prepare mode (ignoring --hadoop, --lily, --solr).");
System.out.println("Will start up, stop, and then snapshot the data directory.");
System.out.println("Please be patient.");
System.out.println("-------------------------------------------------------------");
// If there would be an old template dir, drop it
TemplateDir.deleteTemplateDir();
}
if (enableHadoop) {
enabledServices.add(hadoopService);
}
if (enableSolr) {
enabledServices.add(solrService);
}
if (enableLily) {
enabledServices.add(lilyService);
}
if (enableHbaseIndexer) {
enabledServices.add(hbaseIndexerLauncherService);
}
//
// Determine directory below which all services will store their data
//
if (!prepareMode && cmd.hasOption(dataDirOption.getOpt())) {
String dataDir = cmd.getOptionValue(dataDirOption.getOpt());
testHome = new File(dataDir);
if (testHome.exists() && !testHome.isDirectory()) {
System.err.println("Specified data directory exists and is not a directory:");
System.err.println(testHome.getAbsolutePath());
return -1;
} else if (testHome.exists() && testHome.isDirectory() && testHome.list().length > 0) {
System.out.println("Specified data directory exists: will re-use data from previous run!");
} else if (!testHome.exists()) {
FileUtils.forceMkdir(testHome);
}
// If the user specified the storage directory, do not delete it
clearData = false;
} else {
testHome = TestHomeUtil.createTestHome("lily-launcher-");
if (!prepareMode) {
TemplateDir.restoreTemplateDir(testHome);
}
}
//
// Start the services
//
// Informational messages to be outputted after startup has completed.
List<String> postStartupInfo = new ArrayList<String>();
for (LauncherService service : enabledServices) {
if ((result = service.setup(cmd, testHome, clearData)) != 0) {
return result;
}
}
for (LauncherService service : enabledServices) {
if ((result = service.start(postStartupInfo)) != 0) {
return result;
}
}
if (prepareMode) {
System.out.println("----------------------------------------------------------");
System.out.println("Prepare mode: stopping all services");
System.out.println("----------------------------------------------------------");
lilyService.stop();
solrService.stop();
hadoopService.stop();
System.out.println("----------------------------------------------------------");
System.out.println("Prepare mode: creating template data directory");
TemplateDir.makeTemplateDir(testHome);
System.out.println("----------------------------------------------------------");
System.out.println("Done");
System.exit(0);
}
// Register MBean
ManagementFactory.getPlatformMBeanServer().registerMBean(this, new ObjectName("LilyLauncher:name=Launcher"));
// Add shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook()));
//
// Finished startup, print messages
//
for (String msg : postStartupInfo) {
System.out.println(msg);
}
// redirect all jdk logging (e.g. from Restlet) to log4j (done after startup of all services, to make sure
// all loggers registered during startup of some services are also redirected)
JavaLoggingToLog4jRedirector.activate();
return 0;
}
private class ShutdownHook implements Runnable {
@Override
public void run() {
try {
System.out.println("----------------------------------------------------------");
System.out.println("Shutting down");
System.out.println("----------------------------------------------------------");
//
// Attempt to shutdown everything
//
for (LauncherService service : Lists.reverse(enabledServices)) {
service.stop();
}
//
// Cleanup temp data dir
//
if (clearData) {
System.out.println("Deleting " + testHome.getPath());
TestHomeUtil.cleanupTestHome(testHome);
}
System.out.println("Bye!");
} catch (Throwable t) {
log.info("Error in " + getCmdName() + " shutdown hook", t);
}
}
}
private final AtomicBoolean resetRunning = new AtomicBoolean();
@Override
public void resetLilyState() {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
if (!resetRunning.compareAndSet(false, true)) {
throw new RuntimeException("There is already a Lily state reset running.");
}
try {
long before = System.currentTimeMillis();
if (!enableLily || !enableHadoop || !enableSolr) {
throw new Exception("resetLilyState is only supported when all services are running");
}
// Stop Lily
System.out.println("Stopping Lily");
lilyService.stop();
System.out.println("Lily stopped");
// Clear HBase tables
System.out.println("Clearing HBase tables");
CleanupUtil cleanupUtil = new CleanupUtil(hadoopService.getConf(), "localhost:2181");
cleanupUtil.cleanTables();
HConnectionManager.deleteConnection(hadoopService.getConf(), true);
// Clear Lily ZooKeeper state
System.out.println("Clearing Lily's ZooKeeper state");
cleanupUtil.cleanZooKeeper();
// Clear replication stat
List<String> peerIds = cleanupUtil.cleanHBaseReplicas();
for (String peerId : peerIds) {
new ReplicationPeerUtil().waitOnReplicationPeerStopped(peerId);
}
// Clear Blobs on hdfs blobstore
String dfsUri = "hdfs://localhost:8020/lily/blobs";
System.out.println("Clearing HDFS Blob Store on " + dfsUri);
cleanupUtil.cleanBlobStore(new URI(dfsUri));
// Clear Solr state
clearSolrState();
// The following is useful to observe what threads were not stopped properly after stopping Lily
if (System.getProperty("lily.launcher.threaddump-after-lily-stop") != null) {
ReflectionUtils.printThreadInfo(new PrintWriter("launcher-threaddump-after-lily-stop"), "Thread dump");
}
// Start Lily
System.out.println("Starting Lily");
lilyService.start(new ArrayList<String>());
System.out.println("Reset Lily state took " + (System.currentTimeMillis() - before) + " ms");
} catch (Exception e) {
System.out.println("Error while resetting Lily state: " + e.getMessage());
e.printStackTrace();
throw new RuntimeException("Error while resetting Lily state: " + e.getMessage(), e);
} finally {
resetRunning.set(false);
}
}
private void clearSolrState() throws Exception {
for (String core : SolrProxy.getSolrCoreNames()) {
System.out.println("Clearing data from Solr core " + core);
int response = sendSolrUpdateRequest("<update><delete><query>*:*</query></delete><commit/></update>", core);
if (response != 200) {
throw new RuntimeException("Solr delete all docs on core " + core +
": expected 200 status but it is " + response);
}
}
}
private int sendSolrUpdateRequest(String request, String core) throws IOException {
URL url = new URL("http://localhost:8983/solr/" + core + "/update");
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
conn.setDoOutput(true);
conn.setRequestProperty("Content-Type", "text/xml");
OutputStream os = conn.getOutputStream();
os.write(request.getBytes("UTF-8"));
os.close();
int response = conn.getResponseCode();
conn.disconnect();
return response;
}
}