package mil.nga.giat.geowave.format.geotools.vector; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import mil.nga.giat.geowave.core.geotime.store.dimension.GeometryWrapper; import mil.nga.giat.geowave.core.geotime.store.dimension.Time; import mil.nga.giat.geowave.core.index.ByteArrayId; import mil.nga.giat.geowave.core.ingest.GeoWaveData; import mil.nga.giat.geowave.core.ingest.local.LocalFileIngestPlugin; import mil.nga.giat.geowave.core.store.CloseableIterator; import mil.nga.giat.geowave.core.store.adapter.WritableDataAdapter; import mil.nga.giat.geowave.core.store.index.CommonIndexValue; import mil.nga.giat.geowave.core.store.index.PrimaryIndex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.data.simple.SimpleFeatureSource; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.type.Name; import org.opengis.filter.Filter; /** * This plugin is used for ingesting any GeoTools supported file data store from * a local file system directly into GeoWave as GeoTools' SimpleFeatures. It * supports the default configuration of spatial and spatial-temporal indices * and does NOT currently support the capability to stage intermediate data to * HDFS to be ingested using a map-reduce job. */ public class GeoToolsVectorDataStoreIngestPlugin implements LocalFileIngestPlugin<SimpleFeature> { private final static Logger LOGGER = LoggerFactory.getLogger(GeoToolsVectorDataStoreIngestPlugin.class); private final static String PROPERTIES_EXTENSION = ".properties"; private final RetypingVectorDataPlugin retypingPlugin; private final Filter filter; private final List<String> featureTypeNames; public GeoToolsVectorDataStoreIngestPlugin( final Filter filter ) { // by default inherit the types of the original file this( null, filter, new ArrayList<String>()); } public GeoToolsVectorDataStoreIngestPlugin( final RetypingVectorDataPlugin retypingPlugin, final Filter filter, List<String> featureTypeNames ) { // this constructor can be used directly as an extension point for // retyping the original feature data, if the retyping plugin is null, // the data will be ingested as the original type this.retypingPlugin = retypingPlugin; this.filter = filter; this.featureTypeNames = featureTypeNames; } @Override public String[] getFileExtensionFilters() { return new String[] {}; } @Override public void init( final File baseDirectory ) {} private static boolean isPropertiesFile( File file ) { return file.getName().toLowerCase( Locale.ENGLISH).endsWith( PROPERTIES_EXTENSION); } private static DataStore getDataStore( final File file ) throws IOException { final Map<Object, Object> map = new HashMap<>(); if (isPropertiesFile(file)) { try (FileInputStream fis = new FileInputStream( file)) { Properties prop = new Properties(); prop.load(fis); map.putAll(prop); final DataStore dataStore = DataStoreFinder.getDataStore(map); return dataStore; } } map.put( "url", file.toURI().toURL()); final DataStore dataStore = DataStoreFinder.getDataStore(map); return dataStore; } @Override public boolean supportsFile( final File file ) { DataStore dataStore = null; try { dataStore = getDataStore(file); dataStore.dispose(); } catch (final Exception e) { LOGGER.info( "GeoTools was unable to read data source for file '" + file.getAbsolutePath() + "'", e); } return dataStore != null; } @Override public WritableDataAdapter<SimpleFeature>[] getDataAdapters( final String globalVisibility ) { return new WritableDataAdapter[] {}; } @Override public CloseableIterator<GeoWaveData<SimpleFeature>> toGeoWaveData( final File input, final Collection<ByteArrayId> primaryIndexIds, final String visibility ) { DataStore dataStore = null; try { dataStore = getDataStore(input); } catch (Exception e) { LOGGER.error( "Exception getting a datastore instance", e); } if (dataStore != null) { List<Name> names = null; try { names = dataStore.getNames(); } catch (IOException e) { LOGGER.error( "Unable to get feature tpes from datastore '" + input.getAbsolutePath() + "'", e); } if (names == null) { LOGGER.error("Unable to get datatore name"); return null; } final List<SimpleFeatureCollection> featureCollections = new ArrayList<SimpleFeatureCollection>(); for (final Name name : names) { try { if (featureTypeNames != null && !featureTypeNames.isEmpty() && !featureTypeNames.contains(name.getLocalPart())) { continue; } final SimpleFeatureSource source = dataStore.getFeatureSource(name); final SimpleFeatureCollection featureCollection; // we pass the filter in here so that the datastore may be // able to take advantage of the filter // but also send the filter along to be evaluated per // feature in case the filter is not respected by the // underlying store, we may want to consider relying on the // filtering being done by the store here if (filter != null) { featureCollection = source.getFeatures(filter); } else { featureCollection = source.getFeatures(); } featureCollections.add(featureCollection); } catch (final Exception e) { LOGGER.error( "Unable to ingest data source for feature name '" + name + "'", e); } } return new SimpleFeatureGeoWaveWrapper( featureCollections, primaryIndexIds, visibility, dataStore, retypingPlugin, filter); } LOGGER.error("Unable to get a datastore instance, getDataStore returned null"); return null; } @Override public PrimaryIndex[] getRequiredIndices() { return new PrimaryIndex[] {}; } @Override public Class<? extends CommonIndexValue>[] getSupportedIndexableTypes() { return new Class[] { GeometryWrapper.class, Time.class }; } }