package mil.nga.giat.geowave.service.impl;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.servlet.ServletConfig;
import javax.ws.rs.Consumes;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import com.google.common.io.Files;
import mil.nga.giat.geowave.core.cli.parser.ManualOperationParams;
import mil.nga.giat.geowave.core.ingest.operations.LocalToGeowaveCommand;
import mil.nga.giat.geowave.core.ingest.operations.LocalToMapReduceToGeowaveCommand;
import mil.nga.giat.geowave.core.ingest.operations.options.IngestFormatPluginOptions;
import mil.nga.giat.geowave.core.store.operations.remote.options.DataStorePluginOptions;
import mil.nga.giat.geowave.core.store.operations.remote.options.IndexPluginOptions;
import mil.nga.giat.geowave.service.IngestService;
import mil.nga.giat.geowave.service.ServiceUtils;
@Produces(MediaType.APPLICATION_JSON)
@Path("/ingest")
public class IngestServiceImpl implements
IngestService
{
private final static Logger LOGGER = LoggerFactory.getLogger(IngestServiceImpl.class);
private final Properties serviceProperties;
private final String hdfs;
private final String hdfsBase;
private final String jobTracker;
public IngestServiceImpl(
@Context
final ServletConfig servletConfig ) {
Properties props = null;
try (InputStream is = servletConfig.getServletContext().getResourceAsStream(
servletConfig.getInitParameter("config.properties"))) {
props = ServiceUtils.loadProperties(is);
}
catch (IOException e) {
LOGGER.error(
e.getLocalizedMessage(),
e);
}
hdfs = ServiceUtils.getProperty(
props,
"hdfs");
hdfsBase = ServiceUtils.getProperty(
props,
"hdfsBase");
jobTracker = ServiceUtils.getProperty(
props,
"jobTracker");
serviceProperties = props;
}
@Override
@POST
@Path("/local")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
public Response localIngest(
final FormDataMultiPart multiPart ) {
ingest(
"localingest",
multiPart);
return Response.ok().build();
}
@Override
@POST
@Path("/hdfs")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
public Response hdfsIngest(
final FormDataMultiPart multiPart ) {
return ingest(
"hdfsingest",
multiPart);
}
private Response ingest(
final String ingestMethod,
final FormDataMultiPart multiPart ) {
final List<FormDataBodyPart> fileFields = multiPart.getFields("file");
if (fileFields == null) {
return Response.noContent().build();
}
// read the list of files
final List<FormDataBodyPart> fields = fileFields;
final Map<String, InputStream> fileMap = new HashMap<String, InputStream>();
for (final FormDataBodyPart field : fields) {
fileMap.put(
field.getFormDataContentDisposition().getFileName(),
field.getValueAs(InputStream.class));
}
final String storeName = multiPart.getField(
"store").getValue();
final String visibility = (multiPart.getField("visibility") != null) ? multiPart.getField(
"visibility").getValue() : null;
final String ingestType = (multiPart.getField("ingestFormat") != null) ? multiPart.getField(
"ingestFormat").getValue() : "geotools-vector";
final String dimType = (multiPart.getField("dimType") != null) ? multiPart.getField(
"dimType").getValue() : "spatial";
final boolean clear = (multiPart.getField("clear") != null) ? Boolean.parseBoolean(multiPart.getField(
"clear").getValue()) : false;
if ((storeName == null) || storeName.isEmpty()) {
throw new WebApplicationException(
Response.status(
Status.BAD_REQUEST).entity(
"Ingest Failed - Missing Store Name").build());
}
final File baseDir = Files.createTempDir();
for (final Map.Entry<String, InputStream> kvp : fileMap.entrySet()) {
final File tempFile = new File(
baseDir,
kvp.getKey());
// read the file
try (OutputStream fileOutputStream = new FileOutputStream(
tempFile)) {
final InputStream inStream = kvp.getValue();
int read = 0;
final byte[] bytes = new byte[1024];
while ((read = inStream.read(bytes)) != -1) {
fileOutputStream.write(
bytes,
0,
read);
}
}
catch (final IOException e) {
throw new WebApplicationException(
Response.status(
Status.INTERNAL_SERVER_ERROR).entity(
"Ingest Failed" + e.getMessage()).build());
}
}
// ingest the files
return runIngest(
baseDir,
ingestMethod,
ingestType,
dimType,
storeName,
visibility,
clear);
}
private Response runIngest(
final File baseDir,
final String ingestMethod,
final String ingestType,
final String dimType,
final String storeName,
final String visibility,
final boolean clear ) {
// Ingest Formats
final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
ingestFormatOptions.selectPlugin(ingestType);
// Indexes
final IndexPluginOptions indexOption = new IndexPluginOptions();
indexOption.selectPlugin(dimType);
// Store
final String namespace = DataStorePluginOptions.getStoreNamespace(storeName);
final DataStorePluginOptions dataStorePlugin = new DataStorePluginOptions();
if (!dataStorePlugin.load(
serviceProperties,
namespace)) {
throw new WebApplicationException(
Response.status(
Status.BAD_REQUEST).entity(
"Ingest Failed - Invalid Store").build());
}
switch (ingestMethod) {
default:
case "localingest":
final LocalToGeowaveCommand localIngester = new LocalToGeowaveCommand();
localIngester.setPluginFormats(ingestFormatOptions);
localIngester.setInputIndexOptions(Arrays.asList(indexOption));
localIngester.setInputStoreOptions(dataStorePlugin);
localIngester.getIngestOptions().setVisibility(
visibility);
localIngester.setParameters(
baseDir.getAbsolutePath(),
null,
null);
localIngester.prepare(new ManualOperationParams());
localIngester.execute(new ManualOperationParams());
return Response.ok().build();
case "hdfsingest":
final LocalToMapReduceToGeowaveCommand hdfsIngester = new LocalToMapReduceToGeowaveCommand();
hdfsIngester.setPluginFormats(ingestFormatOptions);
hdfsIngester.setInputIndexOptions(Arrays.asList(indexOption));
hdfsIngester.setInputStoreOptions(dataStorePlugin);
hdfsIngester.getIngestOptions().setVisibility(
visibility);
hdfsIngester.setParameters(
baseDir.getAbsolutePath(),
hdfs,
hdfsBase,
null,
null);
hdfsIngester.getMapReduceOptions().setJobTrackerHostPort(
jobTracker);
hdfsIngester.prepare(new ManualOperationParams());
hdfsIngester.execute(new ManualOperationParams());
return Response.ok().build();
}
}
}