/**
* (c) Copyright 2012 WibiData, Inc.
*
* See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.kiji.mapreduce;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.kiji.annotations.ApiAudience;
import org.kiji.annotations.ApiStability;
import org.kiji.mapreduce.framework.MapReduceJobBuilder;
import org.kiji.mapreduce.kvstore.KeyValueStore;
import org.kiji.mapreduce.kvstore.KeyValueStoreClient;
/**
* Builds general MapReduce jobs around Kiji mappers and reducers. For specialized tasks see
* {@link org.kiji.mapreduce.produce.KijiProduceJobBuilder},
* {@link org.kiji.mapreduce.gather.KijiGatherJobBuilder},
* {@link org.kiji.mapreduce.bulkimport.KijiBulkImportJobBuilder}.
*/
@ApiAudience.Public
@ApiStability.Stable
public final class KijiMapReduceJobBuilder extends MapReduceJobBuilder<KijiMapReduceJobBuilder> {
private static final Logger LOG = LoggerFactory.getLogger(KijiMapReduceJobBuilder.class);
/** The class of the mapper to run. */
@SuppressWarnings("rawtypes")
private Class<? extends KijiMapper> mMapperClass;
/** The class of the combiner to run. */
@SuppressWarnings("rawtypes")
private Class<? extends KijiReducer> mCombinerClass;
/** The class of the reducer to run. */
@SuppressWarnings("rawtypes")
private Class<? extends KijiReducer> mReducerClass;
/** The mapper instance. */
private KijiMapper<?, ?, ?, ?> mMapper;
/** The combiner instance (may be null if no combiner is specified). */
private KijiReducer<?, ?, ?, ?> mCombiner;
/** The reducer instance (may be null if no reducer is specified). */
private KijiReducer<?, ?, ?, ?> mReducer;
/** The job input. */
private MapReduceJobInput mJobInput;
/** Constructs a builder for jobs that run a MapReduce job to transform data. */
private KijiMapReduceJobBuilder() {
mMapperClass = null;
mCombinerClass = null;
mReducerClass = null;
mMapper = null;
mCombiner = null;
mReducer = null;
mJobInput = null;
}
/**
* Creates a new builder for Kiji transform jobs.
*
* @return a new Kiji transform job builder.
*/
public static KijiMapReduceJobBuilder create() {
return new KijiMapReduceJobBuilder();
}
/**
* Configures the job with input.
*
* @param jobInput The input for the job.
* @return This builder instance so you may chain configuration method calls.
*/
public KijiMapReduceJobBuilder withInput(MapReduceJobInput jobInput) {
mJobInput = jobInput;
return this;
}
/**
* Configures the job with a mapper to run in the map phase.
*
* @param mapperClass The mapper class.
* @return This builder instance so you may chain configuration method calls.
*/
@SuppressWarnings("rawtypes")
public KijiMapReduceJobBuilder withMapper(Class<? extends KijiMapper> mapperClass) {
mMapperClass = mapperClass;
return this;
}
/**
* Configures the job with a combiner to run (optional).
*
* @param combinerClass The combiner class.
* @return This builder instance so you may chain configuration method calls.
*/
@SuppressWarnings("rawtypes")
public KijiMapReduceJobBuilder withCombiner(Class<? extends KijiReducer> combinerClass) {
mCombinerClass = combinerClass;
return this;
}
/**
* Configures the job with a reducer to run (optional).
*
* @param reducerClass The reducer class.
* @return This builder instance so you may chain configuration method calls.
*/
@SuppressWarnings("rawtypes")
public KijiMapReduceJobBuilder withReducer(Class<? extends KijiReducer> reducerClass) {
mReducerClass = reducerClass;
return this;
}
/** {@inheritDoc} */
@Override
protected void configureJob(Job job) throws IOException {
// Check that job input was configured.
if (null == mJobInput) {
throw new JobConfigurationException("Must specify job input.");
}
// Construct the mapper instance.
if (null == mMapperClass) {
throw new JobConfigurationException("Must specify a mapper.");
}
mMapper = ReflectionUtils.newInstance(mMapperClass, job.getConfiguration());
// Construct the combiner instance (if specified).
if (null != mCombinerClass) {
mCombiner = ReflectionUtils.newInstance(mCombinerClass, job.getConfiguration());
}
// Construct the reducer instance (if specified).
if (null != mReducerClass) {
mReducer = ReflectionUtils.newInstance(mReducerClass, job.getConfiguration());
}
StringBuilder name = new StringBuilder();
name.append("Kiji transform: ");
name.append(mMapperClass.getSimpleName());
if (null != mReducerClass) {
name.append(" / ");
name.append(mReducerClass.getSimpleName());
}
job.setJobName(name.toString());
// Configure the MapReduce job.
super.configureJob(job);
}
/** {@inheritDoc} */
@Override
protected Map<String, KeyValueStore<?, ?>> getRequiredStores() {
Map<String, KeyValueStore<?, ?>> requiredStores = new HashMap<String, KeyValueStore<?, ?>>();
if (mMapper instanceof KeyValueStoreClient) {
Map<String, KeyValueStore<?, ?>> mapperStores =
((KeyValueStoreClient) mMapper).getRequiredStores();
if (null != mapperStores) {
mergeStores(requiredStores, mapperStores);
}
}
if (null != mCombiner && mCombiner instanceof KeyValueStoreClient) {
Map<String, KeyValueStore<?, ?>> combinerStores =
((KeyValueStoreClient) mCombiner).getRequiredStores();
if (null != combinerStores) {
mergeStores(requiredStores, combinerStores);
}
}
if (null != mReducer && mReducer instanceof KeyValueStoreClient) {
Map<String, KeyValueStore<?, ?>> reducerStores =
((KeyValueStoreClient) mReducer).getRequiredStores();
if (null != reducerStores) {
mergeStores(requiredStores, reducerStores);
}
}
return requiredStores;
}
/** {@inheritDoc} */
@Override
protected KijiMapReduceJob build(Job job) {
return KijiMapReduceJob.create(job);
}
/** {@inheritDoc} */
@Override
protected MapReduceJobInput getJobInput() {
return mJobInput;
}
/** {@inheritDoc} */
@Override
protected KijiMapper<?, ?, ?, ?> getMapper() {
return mMapper;
}
/** {@inheritDoc} */
@Override
protected KijiReducer<?, ?, ?, ?> getCombiner() {
return mCombiner;
}
/** {@inheritDoc} */
@Override
protected KijiReducer<?, ?, ?, ?> getReducer() {
return mReducer;
}
/** {@inheritDoc} */
@Override
protected Class<?> getJarClass() {
return mMapperClass;
}
}