package mil.nga.giat.geowave.analytic.mapreduce;
import java.io.FileInputStream;
import java.io.IOException;
import mil.nga.giat.geowave.analytic.PropertyManagement;
import mil.nga.giat.geowave.analytic.param.MapReduceParameters;
import mil.nga.giat.geowave.analytic.param.MapReduceParameters.MRConfig;
import mil.nga.giat.geowave.mapreduce.GeoWaveConfiguratorBase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class encapsulates the command-line options and parsed values specific
* to staging intermediate data to HDFS.
*/
public class HadoopOptions
{
private final static Logger LOGGER = LoggerFactory.getLogger(HadoopOptions.class);
private final String hdfsHostPort;
private final Path basePath;
private final String jobTrackerHostPort;
private final Configuration config = new Configuration();
public HadoopOptions(
final PropertyManagement runTimeProperties )
throws IOException {
final boolean setRemoteInvocation = runTimeProperties.hasProperty(MRConfig.HDFS_HOST_PORT)
|| runTimeProperties.hasProperty(MRConfig.JOBTRACKER_HOST_PORT);
final String hostport = runTimeProperties.getPropertyAsString(
MRConfig.HDFS_HOST_PORT,
"localhost:53000");
hdfsHostPort = (!hostport.contains("://")) ? "hdfs://" + hostport : hostport;
basePath = new Path(
runTimeProperties.getPropertyAsString(MRConfig.HDFS_BASE_DIR),
"/");
jobTrackerHostPort = runTimeProperties.getPropertyAsString(
MRConfig.JOBTRACKER_HOST_PORT,
runTimeProperties.getPropertyAsString(MRConfig.YARN_RESOURCE_MANAGER));
final String name = runTimeProperties.getPropertyAsString(MapReduceParameters.MRConfig.CONFIG_FILE);
if (name != null) {
try (FileInputStream in = new FileInputStream(
name)) {
// HP Fortify "Path Manipulation" false positive
// What fortify identifies as "user input" comes
// only from users with OS-level access anyway
config.addResource(
in,
name);
}
catch (final IOException ex) {
LOGGER.error(
"Configuration file not found",
ex);
throw ex;
}
}
if (setRemoteInvocation) {
GeoWaveConfiguratorBase.setRemoteInvocationParams(
hdfsHostPort,
jobTrackerHostPort,
config);
}
else {
LOGGER.info("Assuming local job submission");
}
final FileSystem fs = FileSystem.get(config);
if (!fs.exists(basePath)) {
LOGGER.error("HDFS base directory does not exist");
return;
}
}
public HadoopOptions(
final String hdfsHostPort,
final Path basePath,
final String jobTrackerHostport ) {
this.hdfsHostPort = hdfsHostPort;
this.basePath = basePath;
jobTrackerHostPort = jobTrackerHostport;
}
public String getHdfsHostPort() {
return hdfsHostPort;
}
public Path getBasePath() {
return basePath;
}
public String getJobTrackerOrResourceManagerHostPort() {
return jobTrackerHostPort;
}
public Configuration getConfiguration() {
return config;
}
}