GeoWaveAnalyticJobRunner.java

package mil.nga.giat.geowave.analytic.mapreduce;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.util.Tool;
import org.geotools.feature.type.BasicFeatureTypes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import mil.nga.giat.geowave.analytic.AnalyticFeature;
import mil.nga.giat.geowave.analytic.IndependentJobRunner;
import mil.nga.giat.geowave.analytic.PropertyManagement;
import mil.nga.giat.geowave.analytic.ScopedJobConfiguration;
import mil.nga.giat.geowave.analytic.clustering.ClusteringUtils;
import mil.nga.giat.geowave.analytic.param.FormatConfiguration;
import mil.nga.giat.geowave.analytic.param.InputParameters;
import mil.nga.giat.geowave.analytic.param.OutputParameters;
import mil.nga.giat.geowave.analytic.param.OutputParameters.Output;
import mil.nga.giat.geowave.analytic.param.ParameterEnum;
import mil.nga.giat.geowave.analytic.param.StoreParameters;
import mil.nga.giat.geowave.analytic.param.StoreParameters.StoreParam;
import mil.nga.giat.geowave.analytic.store.PersistableStore;
import mil.nga.giat.geowave.core.geotime.ingest.SpatialDimensionalityTypeProvider;
import mil.nga.giat.geowave.core.index.ByteArrayId;
import mil.nga.giat.geowave.core.store.adapter.AdapterStore;
import mil.nga.giat.geowave.core.store.adapter.DataAdapter;
import mil.nga.giat.geowave.core.store.index.CustomIdIndex;
import mil.nga.giat.geowave.core.store.index.IndexStore;
import mil.nga.giat.geowave.core.store.index.PrimaryIndex;
import mil.nga.giat.geowave.mapreduce.JobContextAdapterStore;
import mil.nga.giat.geowave.mapreduce.JobContextIndexStore;

/**
 * This class managers the input and output formats for a map reduce job. It
 * also controls job submission, isolating some of the job management
 * responsibilities. One key benefit is support of unit testing for job runner
 * instances.
 */
public abstract class GeoWaveAnalyticJobRunner extends
		Configured implements
		Tool,
		MapReduceJobRunner,
		IndependentJobRunner
{

	protected static final Logger LOGGER = LoggerFactory.getLogger(GeoWaveAnalyticJobRunner.class);

	private FormatConfiguration inputFormat = null;
	private FormatConfiguration outputFormat = null;
	private int reducerCount = 1;
	private MapReduceIntegration mapReduceIntegrater = new ToolRunnerMapReduceIntegration();
	private Counters lastCounterSet = null;

	public FormatConfiguration getInputFormatConfiguration() {
		return inputFormat;
	}

	public void setInputFormatConfiguration(
			final FormatConfiguration inputFormat ) {
		this.inputFormat = inputFormat;
	}

	public FormatConfiguration getOutputFormatConfiguration() {
		return outputFormat;
	}

	public void setOutputFormatConfiguration(
			final FormatConfiguration outputFormat ) {
		this.outputFormat = outputFormat;
	}

	public MapReduceIntegration getMapReduceIntegrater() {
		return mapReduceIntegrater;
	}

	public void setMapReduceIntegrater(
			final MapReduceIntegration mapReduceIntegrater ) {
		this.mapReduceIntegrater = mapReduceIntegrater;
	}

	public int getReducerCount() {
		return reducerCount;
	}

	public void setReducerCount(
			final int reducerCount ) {
		this.reducerCount = reducerCount;
	}

	public GeoWaveAnalyticJobRunner() {}

	protected static Logger getLogger() {
		return LOGGER;
	}

	public Class<?> getScope() {
		return this.getClass();
	}

	public AdapterStore getAdapterStore(
			final PropertyManagement runTimeProperties )
			throws Exception {
		PersistableStore store = (PersistableStore) StoreParameters.StoreParam.INPUT_STORE.getHelper().getValue(
				runTimeProperties);
		return store.getDataStoreOptions().createAdapterStore();
	}

	public IndexStore getIndexStore(
			final PropertyManagement runTimeProperties )
			throws Exception {
		PersistableStore store = (PersistableStore) StoreParameters.StoreParam.INPUT_STORE.getHelper().getValue(
				runTimeProperties);
		return store.getDataStoreOptions().createIndexStore();
	}

	@Override
	public int run(
			final Configuration configuration,
			final PropertyManagement runTimeProperties )
			throws Exception {

		if ((inputFormat == null) && runTimeProperties.hasProperty(InputParameters.Input.INPUT_FORMAT)) {
			inputFormat = runTimeProperties.getClassInstance(
					InputParameters.Input.INPUT_FORMAT,
					FormatConfiguration.class,
					null);
		}
		if (inputFormat != null) {
			InputParameters.Input.INPUT_FORMAT.getHelper().setValue(
					configuration,
					getScope(),
					inputFormat.getClass());
			inputFormat.setup(
					runTimeProperties,
					configuration);
		}
		if ((outputFormat == null) && runTimeProperties.hasProperty(OutputParameters.Output.OUTPUT_FORMAT)) {
			outputFormat = runTimeProperties.getClassInstance(
					OutputParameters.Output.OUTPUT_FORMAT,
					FormatConfiguration.class,
					null);
		}

		if (outputFormat != null) {
			OutputParameters.Output.OUTPUT_FORMAT.getHelper().setValue(
					configuration,
					getScope(),
					outputFormat.getClass());
			outputFormat.setup(
					runTimeProperties,
					configuration);
		}

		runTimeProperties.setConfig(
				new ParameterEnum[] {
					StoreParam.INPUT_STORE
				},
				configuration,
				getScope());

		OutputParameters.Output.REDUCER_COUNT.getHelper().setValue(
				configuration,
				getScope(),
				runTimeProperties.getPropertyAsInt(
						OutputParameters.Output.REDUCER_COUNT,
						reducerCount));
		return mapReduceIntegrater.submit(
				configuration,
				runTimeProperties,
				this);
	}

	public static void addDataAdapter(
			final Configuration config,
			final DataAdapter<?> adapter ) {
		JobContextAdapterStore.addDataAdapter(
				config,
				adapter);
	}

	public static void addIndex(
			final Configuration config,
			final PrimaryIndex index ) {
		JobContextIndexStore.addIndex(
				config,
				index);
	}

	@SuppressWarnings("rawtypes")
	@Override
	public int run(
			final String[] args )
			throws Exception {
		final Job job = mapReduceIntegrater.getJob(this);

		configure(job);

		final ScopedJobConfiguration configWrapper = new ScopedJobConfiguration(
				job.getConfiguration(),
				getScope());

		final FormatConfiguration inputFormat = configWrapper.getInstance(
				InputParameters.Input.INPUT_FORMAT,
				FormatConfiguration.class,
				null);

		if (inputFormat != null) {
			job.setInputFormatClass((Class<? extends InputFormat>) inputFormat.getFormatClass());
		}

		final FormatConfiguration outputFormat = configWrapper.getInstance(
				OutputParameters.Output.OUTPUT_FORMAT,
				FormatConfiguration.class,
				null);

		if (outputFormat != null) {
			job.setOutputFormatClass((Class<? extends OutputFormat>) outputFormat.getFormatClass());
		}

		job.setNumReduceTasks(configWrapper.getInt(
				OutputParameters.Output.REDUCER_COUNT,
				1));

		job.setJobName(getJobName());

		job.setJarByClass(this.getClass());
		final Counters counters = mapReduceIntegrater.waitForCompletion(job);
		lastCounterSet = counters;
		return (counters == null) ? 1 : 0;
	}

	abstract protected String getJobName();

	public long getCounterValue(
			final Enum<?> counterEnum ) {
		return (lastCounterSet != null) ? (lastCounterSet.findCounter(counterEnum)).getValue() : 0;
	}

	public abstract void configure(
			final Job job )
			throws Exception;

	@Override
	public Collection<ParameterEnum<?>> getParameters() {
		final List<ParameterEnum<?>> params = new ArrayList<ParameterEnum<?>>();
		if (inputFormat != null) {
			params.addAll(inputFormat.getParameters());
		}
		if (outputFormat != null) {
			params.addAll(outputFormat.getParameters());
		}
		params.addAll(Arrays.asList(new ParameterEnum<?>[] {
			StoreParam.INPUT_STORE,
			Output.REDUCER_COUNT,
			Output.OUTPUT_FORMAT
		}));
		return params;
	}

	@Override
	public int run(
			final PropertyManagement runTimeProperties )
			throws Exception {
		return this.run(
				mapReduceIntegrater.getConfiguration(runTimeProperties),
				runTimeProperties);
	}

	protected DataAdapter<?> getAdapter(
			final PropertyManagement runTimeProperties,
			final ParameterEnum dataTypeEnum,
			final ParameterEnum dataNameSpaceEnum )
			throws Exception {

		final String projectionDataTypeId = runTimeProperties.storeIfEmpty(
				dataTypeEnum,
				"convex_hull").toString();

		final AdapterStore adapterStore = getAdapterStore(runTimeProperties);

		DataAdapter<?> adapter = adapterStore.getAdapter(new ByteArrayId(
				projectionDataTypeId));

		if (adapter == null) {
			final String namespaceURI = runTimeProperties.storeIfEmpty(
					dataNameSpaceEnum,
					BasicFeatureTypes.DEFAULT_NAMESPACE).toString();
			adapter = AnalyticFeature.createGeometryFeatureAdapter(
					projectionDataTypeId,
					new String[0],
					namespaceURI,
					ClusteringUtils.CLUSTERING_CRS);

			adapterStore.addAdapter(adapter);
		}
		return adapter;
	}

	protected String checkIndex(
			final PropertyManagement runTimeProperties,
			final ParameterEnum indexIdEnum,
			final String defaultIdxName )
			throws Exception {

		final String indexId = runTimeProperties.getPropertyAsString(
				indexIdEnum,
				defaultIdxName);

		final IndexStore indexStore = getIndexStore(runTimeProperties);

		PrimaryIndex index = (PrimaryIndex) indexStore.getIndex(new ByteArrayId(
				indexId));
		if (index == null) {
			final PrimaryIndex defaultSpatialIndex = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
			index = new CustomIdIndex(
					defaultSpatialIndex.getIndexStrategy(),
					defaultSpatialIndex.getIndexModel(),
					new ByteArrayId(
							indexId));
			indexStore.addIndex(index);
		}
		return indexId;
	}

}