/* (c) 2017 Open Source Geospatial Foundation - all rights reserved
* This code is licensed under the GPL 2.0 license, available at the root
* application directory.
*/
package org.geoserver.importer.rest;
import net.sf.json.JSONException;
import net.sf.json.JSONObject;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.io.IOUtils;
import org.geoserver.catalog.*;
import org.geoserver.catalog.impl.LayerInfoImpl;
import org.geoserver.catalog.impl.StoreInfoImpl;
import org.geoserver.importer.*;
import org.geoserver.importer.rest.converters.ImportJSONReader;
import org.geoserver.importer.rest.converters.ImportJSONWriter;
import org.geoserver.importer.transform.TransformChain;
import org.geoserver.rest.PutIgnoringExtensionContentNegotiationStrategy;
import org.geoserver.rest.RequestInfo;
import org.geoserver.rest.RestBaseController;
import org.geoserver.rest.RestException;
import org.geoserver.rest.util.MediaTypeExtensions;
import org.geotools.referencing.CRS;
import org.geotools.util.logging.Logging;
import org.opengis.referencing.FactoryException;
import org.opengis.referencing.NoSuchAuthorityCodeException;
import org.opengis.referencing.crs.CoordinateReferenceSystem;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.mvc.condition.PatternsRequestCondition;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@RestController
@RequestMapping(path = RestBaseController.ROOT_PATH+"/imports/{id}/tasks", produces = {
MediaType.APPLICATION_JSON_VALUE, MediaType.TEXT_HTML_VALUE })
public class ImportTaskController extends ImportBaseController {
static final Logger LOGGER = Logging.getLogger(ImportTaskController.class);
@Autowired
protected ImportTaskController(Importer importer) {
super(importer);
}
@GetMapping
public ImportWrapper tasksGet(@PathVariable Long id, @RequestParam(required=false) String expand) {
return (writer, builder, converter) -> converter.tasks(builder,context(id).getTasks(), true, converter.expand(expand, 0));
}
@GetMapping(path = "/{taskId}")
public ImportTask taskGet(@PathVariable Long id, @PathVariable Integer taskId) {
return task(id, taskId, false);
}
@GetMapping(path = "/{taskId}/progress")
public ImportWrapper progressGet(@PathVariable Long id, @PathVariable Integer taskId) {
JSONObject progress = new JSONObject();
ImportTask inProgress = importer.getCurrentlyProcessingTask(id);
try {
if (inProgress != null) {
progress.put("progress", inProgress.getNumberProcessed());
progress.put("total", inProgress.getTotalToProcess());
progress.put("state", inProgress.getState().toString());
} else {
ImportTask task = task(id, taskId);
progress.put("state", task.getState().toString());
if (task.getState() == ImportTask.State.ERROR) {
if (task.getError() != null) {
progress.put("message", task.getError().getMessage());
}
}
}
} catch (JSONException jex) {
throw new RestException("Internal Error", HttpStatus.INTERNAL_SERVER_ERROR, jex);
}
return (writer,builder,converter) -> writer.write(progress.toString());
}
@GetMapping(path = "/{taskId}/target")
public ImportWrapper targetGet(@PathVariable Long id, @PathVariable Integer taskId, @RequestParam(required=false) String expand) {
final ImportTask task = task(id, taskId);
if (task.getStore() == null) {
throw new RestException("Task has no target store", HttpStatus.NOT_FOUND);
}
return (writer, builder, converter) -> converter.store(builder,task.getStore(), task, true, converter.expand(expand, 1));
}
@GetMapping(path = "/{taskId}/layer")
public ImportWrapper layersGet(@PathVariable Long id, @PathVariable Integer taskId, @RequestParam(required=false) String expand) {
ImportTask task = task(id, taskId);
return (writer, builder, converter) -> converter.layer(builder,task, true, converter.expand(expand, 1));
}
@PostMapping(consumes = {MediaType.MULTIPART_FORM_DATA_VALUE, MediaType.APPLICATION_FORM_URLENCODED_VALUE})
public Object taskPost(@PathVariable Long id, @RequestParam(required=false) String expand, HttpServletRequest request, HttpServletResponse response) {
ImportData data = null;
LOGGER.info("Handling POST of " + request.getContentType());
//file posted from form
MediaType mimeType = MediaType.valueOf(request.getContentType());
if (request.getContentType().startsWith(MediaType.MULTIPART_FORM_DATA_VALUE)) {
data = handleMultiPartFormUpload(request, context(id));
} else if (request.getContentType().startsWith(MediaType.APPLICATION_FORM_URLENCODED_VALUE)) {
try {
data = handleFormPost(request);
} catch (IOException | ServletException e) {
throw new RestException(e.getMessage(), HttpStatus.INTERNAL_SERVER_ERROR, e);
}
}
if (data == null) {
throw new RestException("Unsupported POST", HttpStatus.FORBIDDEN);
}
//Construct response
return acceptData(data, context(id), response, expand);
}
@PutMapping(path = "/{taskId}", consumes = {MediaType.APPLICATION_JSON_VALUE, MediaTypeExtensions.TEXT_JSON_VALUE})
public ImportWrapper taskPut(
@PathVariable Long id,
@PathVariable Integer taskId,
HttpServletRequest request, HttpServletResponse response) {
return (writer, builder, converter) -> handleTaskPut(id, taskId, request, response, converter);
}
/**
* Workaround to support regular response content type when file extension is in path
*/
@Configuration
static class ImportTaskControllerConfiguration {
@Bean
PutIgnoringExtensionContentNegotiationStrategy importTaskPutContentNegotiationStrategy() {
return new PutIgnoringExtensionContentNegotiationStrategy(
new PatternsRequestCondition("/imports/{id}/tasks/{taskId:.+}"),
Arrays.asList(MediaType.APPLICATION_JSON, MediaType.TEXT_HTML));
}
}
@PutMapping(path = "/{taskId:.+}")
public Object taskPutFile(
@PathVariable Long id,
@PathVariable Object taskId,
@RequestParam(required=false) String expand,
HttpServletRequest request, HttpServletResponse response) {
ImportContext context = context(id);
//TODO: Task id is the file name here. This functionality is completely undocumented
return acceptData(handleFileUpload(context, taskId, request), context, response, expand);
}
@PutMapping(path = "/{taskId}/target")
@ResponseStatus(HttpStatus.NO_CONTENT)
public void targetPut(
@PathVariable Long id,
@PathVariable Integer taskId,
@RequestBody StoreInfo store) {
if (store == null) {
throw new RestException("Task has no target store", HttpStatus.NOT_FOUND);
} else {
updateStoreInfo(task(id, taskId), store, importer);
importer.changed(task(id, taskId));
}
}
@PutMapping(path = "/{taskId}/layer")
@ResponseStatus(HttpStatus.ACCEPTED)
public ImportWrapper layerPut(@PathVariable Long id, @PathVariable Integer taskId, @RequestParam(required=false) String expand, @RequestBody LayerInfo layer) {
ImportTask task = task(id, taskId);
return (writer, builder, converter) -> {
updateLayer(task, layer, importer, converter);
importer.changed(task);
converter.task(builder,task, true, converter.expand(expand, 1));
};
}
@DeleteMapping(path = "/{taskId}")
@ResponseStatus(HttpStatus.NO_CONTENT)
public void taskDelete(@PathVariable Long id, @PathVariable Integer taskId) {
ImportTask task = task(id, taskId);
task.getContext().removeTask(task);
importer.changed(task.getContext());
}
public Object acceptData(ImportData data, ImportContext context, HttpServletResponse response, String expand) {
List<ImportTask> newTasks = null;
try {
newTasks = importer.update(context, data);
} catch (ValidationException ve) {
throw ImportJSONWriter.badRequest(ve.getMessage());
} catch (IOException e) {
throw new RestException("Error updating context", HttpStatus.INTERNAL_SERVER_ERROR, e);
}
if (!newTasks.isEmpty()) {
final List<ImportTask> result = newTasks;
if (newTasks.size() == 1) {
long taskId = newTasks.get(0).getId();
response.setHeader("Location", RequestInfo.get().servletURI(String.format("/imports/%d/tasks/%d", context.getId(), taskId)));
}
response.setStatus(HttpStatus.CREATED.value());
return (ImportWrapper) (writer, builder,converter) -> {
if (result.size() == 1) {
converter.task(builder,result.get(0), true, converter.expand(expand, 1));
}
else {
converter.tasks(builder,result, true, converter.expand(expand, 0));
}
};
}
return "";
}
public ImportData handleFormPost(HttpServletRequest request) throws IOException, ServletException {
String url = IOUtils.toString(request.getPart("url").getInputStream(), encoding);
if (url == null) {
throw new RestException("Invalid request", HttpStatus.BAD_REQUEST);
}
URL location = null;
try {
location = new URL(url);
} catch (MalformedURLException ex) {
LOGGER.warning("invalid URL specified in upload : " + url);
}
// @todo handling remote URL implies asynchronous processing at this stage
if (location == null || !location.getProtocol().equalsIgnoreCase("file")) {
throw new RestException("Invalid url in request", HttpStatus.BAD_REQUEST);
}
FileData file;
try {
file = FileData.createFromFile(new File(location.toURI().getPath()));
} catch (Exception ex) {
throw new RuntimeException("Unexpected exception", ex);
}
if (file instanceof Directory) {
try {
file.prepare();
} catch (IOException ioe) {
String msg = "Error processing file: " + file.getFile().getAbsolutePath();
LOGGER.log(Level.WARNING, msg, ioe);
throw new RestException(msg, HttpStatus.INTERNAL_SERVER_ERROR);
}
}
return file;
}
public ImportData handleMultiPartFormUpload(HttpServletRequest request, ImportContext context) {
DiskFileItemFactory factory = new DiskFileItemFactory();
// @revisit - this appears to be causing OOME
//factory.setSizeThreshold(102400000);
ServletFileUpload upload = new ServletFileUpload(factory);
List<FileItem> items = null;
try {
items = upload.parseRequest(request);
} catch (FileUploadException e) {
throw new RestException("File upload failed", HttpStatus.INTERNAL_SERVER_ERROR, e);
}
//look for a directory to hold the files
Directory directory = findOrCreateDirectory(context);
//unpack all the files
for (FileItem item : items) {
if (item.getName() == null) {
continue;
}
try {
directory.accept(item);
} catch (Exception ex) {
throw new RestException("Error writing file " + item.getName(), HttpStatus.INTERNAL_SERVER_ERROR, ex);
}
}
return directory;
}
void handleTaskPut(Long id, Integer taskId, HttpServletRequest request, HttpServletResponse response, ImportJSONWriter converter) {
ImportTask orig = task(id, taskId);
ImportTask task;
try {
ImportJSONReader reader = new ImportJSONReader(importer);
task = reader.task(request.getInputStream());
// task = new ImportContextJSONConverterReader(importer,request.getInputStream()).task();
} catch (ValidationException | IOException ve) {
LOGGER.log(Level.WARNING, null, ve);
throw converter.badRequest(ve.getMessage());
}
boolean change = false;
if (task.getStore() != null) {
//JD: moved to TaskTargetResource, but handle here for backward compatability
updateStoreInfo(orig, task.getStore(), importer);
change = true;
}
if (task.getData() != null) {
//TODO: move this to data endpoint
orig.getData().setCharsetEncoding(task.getData().getCharsetEncoding());
change = true;
}
if (task.getUpdateMode() != null) {
orig.setUpdateMode(task.getUpdateMode());
change = orig.getUpdateMode() != task.getUpdateMode();
}
if (task.getLayer() != null) {
change = true;
//now handled by LayerResource, but handle here for backwards compatability
updateLayer(orig, task.getLayer(), importer, converter);
}
TransformChain chain = task.getTransform();
if (chain != null) {
orig.setTransform(chain);
change = true;
}
if (!change) {
throw new RestException("Unknown representation", HttpStatus.BAD_REQUEST);
} else {
importer.changed(orig);
response.setStatus(HttpStatus.NO_CONTENT.value());
}
}
private Directory findOrCreateDirectory(ImportContext context) {
if (context.getData() instanceof Directory) {
return (Directory) context.getData();
}
try {
return Directory.createNew(importer.getUploadRoot());
} catch (IOException ioe) {
throw new RestException("File upload failed", HttpStatus.INTERNAL_SERVER_ERROR, ioe);
}
}
private ImportData handleFileUpload(ImportContext context, Object taskId, HttpServletRequest request) {
Directory directory = findOrCreateDirectory(context);
try {
directory.accept(taskId.toString(),request.getInputStream());
} catch (IOException e) {
throw new RestException("Error unpacking file",
HttpStatus.INTERNAL_SERVER_ERROR, e);
}
return directory;
}
static void updateLayer(ImportTask orig, LayerInfo l, Importer importer, ImportJSONWriter converter) {
//update the original layer and resource from the new
ResourceInfo r = l.getResource();
//TODO: this is not thread safe, clone the object before overwriting it
//save the existing resource, which will be overwritten below,
ResourceInfo resource = orig.getLayer().getResource();
if (r != null) {
// we support the following resource info properties:
// (don't just use blindly copy everything)
if (r.getTitle() != null) {
resource.setTitle(r.getTitle());
}
if (r.getAbstract() != null) {
resource.setAbstract(r.getAbstract());
}
if (r.getDescription() != null) {
resource.setDescription(r.getDescription());
}
}
CatalogBuilder cb = new CatalogBuilder(importer.getCatalog());
l.setResource(resource);
// @hack workaround OWSUtils bug - trying to copy null collections
// why these are null in the first place is a different question
LayerInfoImpl impl = (LayerInfoImpl) orig.getLayer();
if (impl.getAuthorityURLs() == null) {
impl.setAuthorityURLs(new ArrayList(1));
}
if (impl.getIdentifiers() == null) {
impl.setIdentifiers(new ArrayList(1));
}
// @endhack
cb.updateLayer(orig.getLayer(), l);
// validate SRS - an invalid one will destroy capabilities doc and make
// the layer totally broken in UI
CoordinateReferenceSystem newRefSystem = null;
String srs = r != null ? r.getSRS() : null;
if (srs != null) {
try {
newRefSystem = CRS.decode(srs);
} catch (NoSuchAuthorityCodeException ex) {
String msg = "Invalid SRS " + srs;
LOGGER.warning(msg + " in PUT request");
throw converter.badRequest(msg);
} catch (FactoryException ex) {
throw new RestException("Error with referencing", HttpStatus.INTERNAL_SERVER_ERROR,ex);
}
// make this the specified native if none exists
// useful for csv or other files
if (resource.getNativeCRS() == null) {
resource.setNativeCRS(newRefSystem);
}
resource.setSRS(srs);
}
}
static void updateStoreInfo(ImportTask task, StoreInfo update, Importer importer) {
//handle three cases here:
// 1. current task store is null -> set the update as the new store
// 2. update is reference to an existing store -> set the update as the new store
// 3. update is a partial change to the current store -> update the current
// allow an existing store to be referenced as the target
StoreInfo orig = task.getStore();
//check if the update is referencing an existing store
StoreInfo existing = null;
if (update.getName() != null) {
Catalog cat = importer.getCatalog();
if (update.getWorkspace() != null) {
existing = cat.getStoreByName(
update.getWorkspace(), update.getName(), StoreInfo.class);
}
else {
existing = importer.getCatalog().getStoreByName(update.getName(), StoreInfo.class);
}
if (existing == null) {
throw new RestException("Unable to find referenced store", HttpStatus.BAD_REQUEST);
}
if (!existing.isEnabled()) {
throw new RestException("Proposed target store is not enabled", HttpStatus.BAD_REQUEST);
}
}
if (existing != null) {
//JD: not sure why we do this, rather than just task.setStore(existing);
CatalogBuilder cb = new CatalogBuilder(importer.getCatalog());
StoreInfo clone;
if (existing instanceof DataStoreInfo) {
clone = cb.buildDataStore(existing.getName());
cb.updateDataStore((DataStoreInfo) clone, (DataStoreInfo) existing);
}
else if (existing instanceof CoverageStoreInfo) {
clone = cb.buildCoverageStore(existing.getName());
cb.updateCoverageStore((CoverageStoreInfo) clone, (CoverageStoreInfo) existing);
}
else {
throw new RestException(
"Unable to handle existing store: " + update, HttpStatus.INTERNAL_SERVER_ERROR);
}
((StoreInfoImpl) clone).setId(existing.getId());
task.setStore(clone);
task.setDirect(false);
}
else if (orig == null){
task.setStore(update);
}
else {
//update the original
CatalogBuilder cb = new CatalogBuilder(importer.getCatalog());
if (orig instanceof DataStoreInfo) {
cb.updateDataStore((DataStoreInfo)orig, (DataStoreInfo)update);
}
else if (orig instanceof CoverageStoreInfo) {
cb.updateCoverageStore((CoverageStoreInfo)orig, (CoverageStoreInfo)update);
}
else {
throw new RestException(
"Unable to update store with " + update, HttpStatus.INTERNAL_SERVER_ERROR);
}
}
}
}