PartitionGraph.java

package edu.umd.cloud9.example.hits;

import java.io.IOException;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Logger;

import edu.umd.cloud9.example.hits.RangePartitioner;

/**
 * <p>
 * Driver program for partitioning the graph. This version reads & writes HITSNode writables. Command-line arguments are as
 * follows:
 * </p>
 * 
 * <ul>
 * 
 * <li>[inputDir]: input directory</li>
 * <li>[outputDir]: output directory</li>
 * <li>[numPartitions]: number of partitions</li>
 * <li>[useRange?]: 1 to use range partitioning or 0 otherwise</li>
 * <li>[nodeCount]: number of nodes in the graph</li>
 * 
 * </ul>
 * 
 * @author Jimmy Lin
 * @author Mike McGrath
 * 
 */

public class PartitionGraph extends Configured implements Tool {
	private static final Logger sLogger = Logger
			.getLogger(PartitionGraph.class);

	private static class MapClass extends MapReduceBase implements
			Mapper<IntWritable, HITSNode, IntWritable, HITSNode> {
		public void map(IntWritable nid, HITSNode node,
				OutputCollector<IntWritable, HITSNode> output, Reporter reporter)
				throws IOException {
			output.collect(nid, node);
		}
	}

	private static class ReduceClass extends MapReduceBase implements
			Reducer<IntWritable, HITSNode, IntWritable, HITSNode> {
		public void reduce(IntWritable nid, Iterator<HITSNode> values,
				OutputCollector<IntWritable, HITSNode> output, Reporter reporter)
				throws IOException {
			while (values.hasNext()) {
				HITSNode node = values.next();
				output.collect(nid, node);
			}
		}
	}

	public static void main(String[] args) throws Exception {
		int res = ToolRunner.run(new Configuration(), new PartitionGraph(),
				args);
		System.exit(res);
	}

	public PartitionGraph() {
	}

	private static int printUsage() {
		System.out
				.println("usage: [in-path] [out-path] [numPartitions] [useRange?] [nodeCount]");
		ToolRunner.printGenericCommandUsage(System.out);
		return -1;
	}

	public int run(String[] args) throws IOException {
		if (args.length != 5) {
			printUsage();
			return -1;
		}

		String inPath = args[0];
		String outPath = args[1];
		int numParts = Integer.parseInt(args[2]);
		boolean useRange = Integer.parseInt(args[3]) != 0;
		int nodeCount = Integer.parseInt(args[4]);

		sLogger.info("Tool name: PartitionGraph");
		sLogger.info(" - in dir: " + inPath);
		sLogger.info(" - out dir: " + outPath);
		sLogger.info(" - numParts: " + numParts);
		sLogger.info(" - useRange: " + useRange);
		sLogger.info(" - nodeCnt: " + nodeCount);

		JobConf conf = new JobConf(PartitionGraph.class);

		conf.setJobName("Partition Graph " + numParts);
		conf.setNumReduceTasks(numParts);

		conf.setInt("mapred.min.split.size", 1024 * 1024 * 1024);
		conf.set("mapred.child.java.opts", "-Xmx2048m");
		conf.setInt("NodeCount", nodeCount);

		FileInputFormat.setInputPaths(conf, new Path(inPath));
		FileOutputFormat.setOutputPath(conf, new Path(outPath));

		conf.setInputFormat(SequenceFileInputFormat.class);
		conf.setOutputFormat(SequenceFileOutputFormat.class);

		conf.setMapOutputKeyClass(IntWritable.class);
		conf.setMapOutputValueClass(HITSNode.class);

		conf.setOutputKeyClass(IntWritable.class);
		conf.setOutputValueClass(HITSNode.class);

		conf.setMapperClass(MapClass.class);
		conf.setReducerClass(ReduceClass.class);

		conf.setSpeculativeExecution(false);

		if (useRange) {
			conf.setPartitionerClass(RangePartitioner.class);
		}

		FileSystem.get(conf).delete(new Path(outPath), true);

		JobClient.runJob(conf);

		return 0;
	}
}