package mil.nga.giat.geowave.cli.osm.operations; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.util.ToolRunner; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; import mil.nga.giat.geowave.cli.osm.mapreduce.Convert.OSMConversionRunner; import mil.nga.giat.geowave.cli.osm.mapreduce.Ingest.OSMRunner; import mil.nga.giat.geowave.cli.osm.operations.options.OSMIngestCommandArgs; import mil.nga.giat.geowave.cli.osm.osmfeature.types.features.FeatureDefinitionSet; import mil.nga.giat.geowave.core.cli.annotations.GeowaveOperation; import mil.nga.giat.geowave.core.cli.api.Command; import mil.nga.giat.geowave.core.cli.api.DefaultOperation; import mil.nga.giat.geowave.core.cli.api.OperationParams; import mil.nga.giat.geowave.core.store.operations.remote.options.DataStorePluginOptions; import mil.nga.giat.geowave.core.store.operations.remote.options.StoreLoader; @GeowaveOperation(name = "ingest", parentOperation = OSMSection.class) @Parameters(commandDescription = "Ingest and convert OSM data from HDFS to GeoWave") public class IngestOSMToGeoWaveCommand extends DefaultOperation implements Command { @Parameter(description = "<hdfs host:port> <path to base directory to read from> <store name>") private List<String> parameters = new ArrayList<String>(); @ParametersDelegate private OSMIngestCommandArgs ingestOptions = new OSMIngestCommandArgs(); private DataStorePluginOptions inputStoreOptions = null; @Override public void execute( OperationParams params ) throws Exception { // Ensure we have all the required arguments if (parameters.size() != 3) { throw new ParameterException( "Requires arguments: <hdfs host:port> <path to base directory to read from> <store name>"); } String hdfsHostPort = parameters.get(0); String basePath = parameters.get(1); String inputStoreName = parameters.get(2); // Ensures that the url starts with hdfs:// if (!hdfsHostPort.contains("://")) { hdfsHostPort = "hdfs://" + hdfsHostPort; } if (!basePath.startsWith("/")) { throw new ParameterException( "HDFS Base path must start with forward slash /"); } // Attempt to load input store. if (inputStoreOptions == null) { StoreLoader inputStoreLoader = new StoreLoader( inputStoreName); if (!inputStoreLoader.loadFromConfig(getGeoWaveConfigFile(params))) { throw new ParameterException( "Cannot find store name: " + inputStoreLoader.getStoreName()); } inputStoreOptions = inputStoreLoader.getDataStorePlugin(); } // Copy over options from main parameter to ingest options ingestOptions.setHdfsBasePath(basePath); ingestOptions.setNameNode(hdfsHostPort); if (inputStoreOptions.getGeowaveNamespace() == null) { inputStoreOptions.getFactoryOptions().setGeowaveNamespace( "osmnamespace"); } if (ingestOptions.getVisibilityOptions().getVisibility() == null) { ingestOptions.getVisibilityOptions().setVisibility( "public"); } // This is needed by a method in OSMIngsetCommandArgs. ingestOptions.setOsmNamespace(inputStoreOptions.getGeowaveNamespace()); // Ingest the data. ingestData(); // Convert the data convertData(); } private void ingestData() throws Exception { OSMRunner runner = new OSMRunner( ingestOptions, inputStoreOptions); int res = ToolRunner.run( runner, new String[] {}); if (res != 0) { throw new RuntimeException( "OSMRunner failed: " + res); } System.out.println("finished ingest"); System.out.println("**************************************************"); } private void convertData() throws Exception { FeatureDefinitionSet.initialize(new OSMIngestCommandArgs().getMappingContents()); OSMConversionRunner runner = new OSMConversionRunner( ingestOptions, inputStoreOptions); int res = ToolRunner.run( runner, new String[] {}); if (res != 0) { throw new RuntimeException( "OSMConversionRunner failed: " + res); } System.out.println("finished conversion"); System.out.println("**************************************************"); System.out.println("**************************************************"); System.out.println("**************************************************"); } public List<String> getParameters() { return parameters; } public void setParameters( String hdfsHostPort, String hdfsPath, String storeName ) { this.parameters = new ArrayList<String>(); this.parameters.add(hdfsHostPort); this.parameters.add(hdfsPath); this.parameters.add(storeName); } public OSMIngestCommandArgs getIngestOptions() { return ingestOptions; } public void setIngestOptions( OSMIngestCommandArgs ingestOptions ) { this.ingestOptions = ingestOptions; } public DataStorePluginOptions getInputStoreOptions() { return inputStoreOptions; } public void setInputStoreOptions( DataStorePluginOptions inputStoreOptions ) { this.inputStoreOptions = inputStoreOptions; } }