/**
* (c) Copyright 2012 WibiData, Inc.
*
* See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.kiji.mapreduce;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.kiji.mapreduce.input.MapReduceJobInputs;
import org.kiji.mapreduce.kvstore.KeyValueStore;
import org.kiji.mapreduce.kvstore.KeyValueStoreClient;
import org.kiji.mapreduce.kvstore.RequiredStores;
import org.kiji.mapreduce.kvstore.framework.KeyValueStoreConfiguration;
import org.kiji.mapreduce.kvstore.impl.KeyValueStoreConfigSerializer;
import org.kiji.mapreduce.kvstore.lib.EmptyKeyValueStore;
import org.kiji.mapreduce.kvstore.lib.SeqFileKeyValueStore;
import org.kiji.mapreduce.output.MapReduceJobOutputs;
import org.kiji.schema.util.Resources;
import org.kiji.schema.util.TestingFileUtils;
public class TestKijiMapReduceJobBuilder {
private static final Logger LOG = LoggerFactory.getLogger(TestKijiMapReduceJobBuilder.class);
/** A line count mapper for testing. */
public static class MyMapper extends KijiMapper<LongWritable, Text, Text, IntWritable>
implements KeyValueStoreClient {
@Override
protected void map(LongWritable offset, Text line, Context context)
throws IOException, InterruptedException {
context.write(line, new IntWritable(1));
}
@Override
public Class<?> getOutputKeyClass() {
return Text.class;
}
@Override
public Class<?> getOutputValueClass() {
return IntWritable.class;
}
@Override
public Map<String, KeyValueStore<?, ?>> getRequiredStores() {
return RequiredStores.just("mapperMap", EmptyKeyValueStore.get());
}
}
/** A sum reducer for testing. */
public static class MyReducer extends KijiReducer<Text, IntWritable, Text, IntWritable>
implements KeyValueStoreClient {
@Override
protected void reduce(Text line, Iterable<IntWritable> counts, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable count : counts) {
sum += count.get();
}
context.write(line, new IntWritable(sum));
}
@Override
public Class<?> getOutputKeyClass() {
return Text.class;
}
@Override
public Class<?> getOutputValueClass() {
return IntWritable.class;
}
@Override
public Map<String, KeyValueStore<?, ?>> getRequiredStores() {
return RequiredStores.with("reducerMap", EmptyKeyValueStore.get());
}
}
private Configuration mConf = null;
private File mTempDir = null;
@Before
public final void setupTestKijiTransformJob() throws Exception {
mTempDir = TestingFileUtils.createTempDir("kijimrtest", "dir");
mConf = new Configuration();
mConf.set("fs.default.name", "file://" + mTempDir);
}
@After
public final void teardownTestKijiTransformJob() throws Exception {
FileUtils.deleteDirectory(mTempDir);
mTempDir = null;
mConf = null;
}
@Test
public void testBuild() throws Exception {
final KijiMapReduceJob job = KijiMapReduceJobBuilder.create()
.withConf(mConf)
.withInput(MapReduceJobInputs.newTextMapReduceJobInput(new Path("/path/to/my/input")))
.withMapper(MyMapper.class)
.withReducer(MyReducer.class)
.withOutput(MapReduceJobOutputs.newTextMapReduceJobOutput(
new Path("/path/to/my/output"), 16))
.build();
final Job hadoopJob = job.getHadoopJob();
assertEquals(TextInputFormat.class, hadoopJob.getInputFormatClass());
assertEquals(MyMapper.class, hadoopJob.getMapperClass());
assertEquals(MyReducer.class, hadoopJob.getReducerClass());
assertEquals(16, hadoopJob.getNumReduceTasks());
assertEquals(TextOutputFormat.class, hadoopJob.getOutputFormatClass());
// KeyValueStore-specific checks here.
Configuration confOut = hadoopJob.getConfiguration();
assertEquals(2, confOut.getInt(KeyValueStoreConfigSerializer.CONF_KEY_VALUE_STORE_COUNT, 0));
assertEquals(EmptyKeyValueStore.class.getName(),
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "0."
+ KeyValueStoreConfigSerializer.CONF_CLASS));
assertEquals("mapperMap",
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "0."
+ KeyValueStoreConfigSerializer.CONF_NAME));
assertEquals(EmptyKeyValueStore.class.getName(),
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "1."
+ KeyValueStoreConfigSerializer.CONF_CLASS));
assertEquals("reducerMap",
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "1."
+ KeyValueStoreConfigSerializer.CONF_NAME));
}
@Test
public void testBuildWithXmlKVStores() throws Exception {
// Test that we can override default configuration KeyValueStores from an XML file.
final InputStream xmlStores =
Resources.openSystemResource("org/kiji/mapreduce/test-kvstores.xml");
// This file needs to exist before we build the job, or else
// we can't build the job; it's referenced by a key-value store that checks
// for its presence.
final File tmpFile = new File("/tmp/foo.seq");
if (tmpFile.createNewFile()) {
// We created this temp file, we're responsible for deleting it.
tmpFile.deleteOnExit();
}
LOG.info("Building job...");
final KijiMapReduceJob job = KijiMapReduceJobBuilder.create()
.withConf(mConf)
.withInput(MapReduceJobInputs.newTextMapReduceJobInput(new Path("/path/to/my/input")))
.withMapper(MyMapper.class)
.withReducer(MyReducer.class)
.withOutput(MapReduceJobOutputs.newTextMapReduceJobOutput(
new Path("/path/to/my/output"), 16))
.withStoreBindings(xmlStores)
.build();
xmlStores.close();
LOG.info("Verifying job configuration...");
final Job hadoopJob = job.getHadoopJob();
assertEquals(TextInputFormat.class, hadoopJob.getInputFormatClass());
assertEquals(MyMapper.class, hadoopJob.getMapperClass());
assertEquals(MyReducer.class, hadoopJob.getReducerClass());
assertEquals(16, hadoopJob.getNumReduceTasks());
assertEquals(TextOutputFormat.class, hadoopJob.getOutputFormatClass());
// KeyValueStore-specific checks here.
// We override mapperMap with a SeqFileKeyValueStore.
Configuration confOut = hadoopJob.getConfiguration();
assertEquals(2, confOut.getInt(KeyValueStoreConfigSerializer.CONF_KEY_VALUE_STORE_COUNT, 0));
assertEquals(SeqFileKeyValueStore.class.getName(),
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "0."
+ KeyValueStoreConfigSerializer.CONF_CLASS));
assertEquals("mapperMap",
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "0."
+ KeyValueStoreConfigSerializer.CONF_NAME));
assertEquals(EmptyKeyValueStore.class.getName(),
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "1."
+ KeyValueStoreConfigSerializer.CONF_CLASS));
assertEquals("reducerMap",
confOut.get(KeyValueStoreConfiguration.KEY_VALUE_STORE_NAMESPACE + "1."
+ KeyValueStoreConfigSerializer.CONF_NAME));
}
@Test
public void testLocalPathJarDir() {
// Tests of MapReduceJobBuilder.addJarDirectory() (For KIJIMR-234)
KijiMapReduceJobBuilder builder = KijiMapReduceJobBuilder.create();
builder.addJarDirectory(new File("/tmp"));
builder.addJarDirectory(new File("/"));
builder.addJarDirectory(new File("."));
builder.addJarDirectory(new File("some/subdir"));
builder.addJarDirectory(new Path("/tmp"));
builder.addJarDirectory(new Path("/"));
builder.addJarDirectory(new Path("."));
builder.addJarDirectory(new Path("./"));
builder.addJarDirectory(new Path("some/subdir"));
builder.addJarDirectory(new Path("./subdir"));
builder.addJarDirectory(new Path(".."));
builder.addJarDirectory(new Path("file:/tmp"));
builder.addJarDirectory(new Path("file:///tmp"));
builder.addJarDirectory(new Path("file:/"));
builder.addJarDirectory(new Path("file:///"));
builder.addJarDirectory("/tmp");
builder.addJarDirectory("/");
builder.addJarDirectory(".");
builder.addJarDirectory("some/subdir");
builder.addJarDirectory("./somedir");
}
}