/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.test;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.mortbay.util.ajax.JSON;
/**
* This class drives the creation of a mini-cluster on the local machine. By
* default, a MiniDFSCluster is spawned on the first available ports that are
* found.
*
* A series of command line flags controls the startup cluster options.
*
* This class can dump a Hadoop configuration and some basic metadata (in JSON)
* into a textfile.
*
* To shutdown the cluster, kill the process.
*
* To run this from the command line, do the following (replacing the jar
* version as appropriate):
*
* $HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/share/hadoop/hdfs/hadoop-hdfs-0.24.0-SNAPSHOT-tests.jar org.apache.hadoop.test.MiniDFSClusterManager -options...
*/
public class MiniDFSClusterManager {
private static final Log LOG =
LogFactory.getLog(MiniDFSClusterManager.class);
private MiniDFSCluster dfs;
private String writeDetails;
private int numDataNodes;
private int nameNodePort;
private StartupOption dfsOpts;
private String writeConfig;
private Configuration conf;
private boolean format;
private static final long SLEEP_INTERVAL_MS = 1000 * 60;
/**
* Creates configuration options object.
*/
@SuppressWarnings("static-access")
private Options makeOptions() {
Options options = new Options();
options
.addOption("datanodes", true, "How many datanodes to start (default 1)")
.addOption("format", false, "Format the DFS (default false)")
.addOption("cmdport", true,
"Which port to listen on for commands (default 0--we choose)")
.addOption("nnport", true, "NameNode port (default 0--we choose)")
.addOption("namenode", true, "URL of the namenode (default "
+ "is either the DFS cluster or a temporary dir)")
.addOption(OptionBuilder
.hasArgs()
.withArgName("property=value")
.withDescription("Options to pass into configuration object")
.create("D"))
.addOption(OptionBuilder
.hasArg()
.withArgName("path")
.withDescription("Save configuration to this XML file.")
.create("writeConfig"))
.addOption(OptionBuilder
.hasArg()
.withArgName("path")
.withDescription("Write basic information to this JSON file.")
.create("writeDetails"))
.addOption(OptionBuilder.withDescription("Prints option help.")
.create("help"));
return options;
}
/**
* Main entry-point.
*/
public void run(String[] args) throws IOException {
if (!parseArguments(args)) {
return;
}
start();
sleepForever();
}
private void sleepForever() {
while (true) {
try {
Thread.sleep(SLEEP_INTERVAL_MS);
if (!dfs.isClusterUp()) {
LOG.info("Cluster is no longer up, exiting");
return;
}
} catch (InterruptedException _) {
// nothing
}
}
}
/**
* Starts DFS as specified in member-variable options. Also writes out
* configuration and details, if requested.
*/
public void start() throws IOException, FileNotFoundException {
dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nameNodePort)
.numDataNodes(numDataNodes)
.startupOption(dfsOpts)
.format(format)
.build();
dfs.waitActive();
LOG.info("Started MiniDFSCluster -- namenode on port "
+ dfs.getNameNodePort());
if (writeConfig != null) {
FileOutputStream fos = new FileOutputStream(new File(writeConfig));
conf.writeXml(fos);
fos.close();
}
if (writeDetails != null) {
Map<String, Object> map = new TreeMap<String, Object>();
if (dfs != null) {
map.put("namenode_port", dfs.getNameNodePort());
}
FileWriter fw = new FileWriter(new File(writeDetails));
fw.write(new JSON().toJSON(map));
fw.close();
}
}
/**
* Parses arguments and fills out the member variables.
* @param args Command-line arguments.
* @return true on successful parse; false to indicate that the
* program should exit.
*/
private boolean parseArguments(String[] args) {
Options options = makeOptions();
CommandLine cli;
try {
CommandLineParser parser = new GnuParser();
cli = parser.parse(options, args);
} catch(ParseException e) {
LOG.warn("options parsing failed: "+e.getMessage());
new HelpFormatter().printHelp("...", options);
return false;
}
if (cli.hasOption("help")) {
new HelpFormatter().printHelp("...", options);
return false;
}
if (cli.getArgs().length > 0) {
for (String arg : cli.getArgs()) {
LOG.error("Unrecognized option: " + arg);
new HelpFormatter().printHelp("...", options);
return false;
}
}
// HDFS
numDataNodes = intArgument(cli, "datanodes", 1);
nameNodePort = intArgument(cli, "nnport", 0);
if (cli.hasOption("format")) {
dfsOpts = StartupOption.FORMAT;
format = true;
} else {
dfsOpts = StartupOption.REGULAR;
format = false;
}
// Runner
writeDetails = cli.getOptionValue("writeDetails");
writeConfig = cli.getOptionValue("writeConfig");
// General
conf = new HdfsConfiguration();
updateConfiguration(conf, cli.getOptionValues("D"));
return true;
}
/**
* Updates configuration based on what's given on the command line.
*
* @param conf2 The configuration object
* @param keyvalues An array of interleaved key value pairs.
*/
private void updateConfiguration(Configuration conf2, String[] keyvalues) {
int num_confs_updated = 0;
if (keyvalues != null) {
for (String prop : keyvalues) {
String[] keyval = prop.split("=", 2);
if (keyval.length == 2) {
conf2.set(keyval[0], keyval[1]);
num_confs_updated++;
} else {
LOG.warn("Ignoring -D option " + prop);
}
}
}
LOG.info("Updated " + num_confs_updated +
" configuration settings from command line.");
}
/**
* Extracts an integer argument with specified default value.
*/
private int intArgument(CommandLine cli, String argName, int defaultValue) {
String o = cli.getOptionValue(argName);
try {
if (o != null) {
return Integer.parseInt(o);
}
} catch (NumberFormatException ex) {
LOG.error("Couldn't parse value (" + o + ") for option "
+ argName + ". Using default: " + defaultValue);
}
return defaultValue;
}
/**
* Starts a MiniDFSClusterManager with parameters drawn from the command line.
*/
public static void main(String[] args) throws IOException {
new MiniDFSClusterManager().run(args);
}
}