/**
* Copyright (C) 2010 - 2016 52°North Initiative for Geospatial Open Source
* Software GmbH
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation.
*
* If the program is linked with libraries which are licensed under one of
* the following licenses, the combination of the program with the linked
* library is not considered a "derivative work" of the program:
*
* • Apache License, version 2.0
* • Apache Software License, version 1.0
* • GNU Lesser General Public License, version 3
* • Mozilla Public License, versions 1.0, 1.1 and 2.0
* • Common Development and Distribution License (CDDL), version 1.0
*
* Therefore the distribution of the program linked with libraries licensed
* under the aforementioned licenses, is permitted by the copyright holders
* if the distribution is compliant with both the GNU General Public
* License version 2 and the aforementioned licenses.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*/
package org.n52.wps.server.r.workspace;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.UUID;
import org.n52.wps.io.IOUtils;
import org.n52.wps.io.data.GenericFileData;
import org.n52.wps.io.data.GenericFileDataConstants;
import org.n52.wps.io.data.GenericFileDataWithGT;
import org.n52.wps.io.data.IData;
import org.n52.wps.io.data.ILiteralData;
import org.n52.wps.io.data.binding.complex.GTRasterDataBinding;
import org.n52.wps.io.data.binding.complex.GTVectorDataBinding;
import org.n52.wps.io.data.binding.complex.GenericFileDataBinding;
import org.n52.wps.io.data.binding.complex.GenericFileDataWithGTBinding;
import org.n52.wps.io.data.binding.literal.AbstractLiteralDataBinding;
import org.n52.wps.io.data.binding.literal.LiteralBooleanBinding;
import org.n52.wps.io.data.binding.literal.LiteralByteBinding;
import org.n52.wps.io.data.binding.literal.LiteralDoubleBinding;
import org.n52.wps.io.data.binding.literal.LiteralFloatBinding;
import org.n52.wps.io.data.binding.literal.LiteralIntBinding;
import org.n52.wps.io.data.binding.literal.LiteralLongBinding;
import org.n52.wps.io.data.binding.literal.LiteralShortBinding;
import org.n52.wps.io.data.binding.literal.LiteralStringBinding;
import org.n52.wps.io.datahandler.generator.GeotiffGenerator;
import org.n52.wps.io.datahandler.parser.GeotiffParser;
import org.n52.wps.server.ExceptionReport;
import org.n52.wps.server.r.data.RDataType;
import org.n52.wps.server.r.data.RDataTypeRegistry;
import org.n52.wps.server.r.data.RTypeDefinition;
import org.n52.wps.server.r.syntax.RAnnotation;
import org.n52.wps.server.r.syntax.RAnnotationException;
import org.n52.wps.server.r.syntax.RAnnotationType;
import org.n52.wps.server.r.syntax.RAttribute;
import org.rosuda.REngine.REXP;
import org.rosuda.REngine.REXPMismatchException;
import org.rosuda.REngine.Rserve.RConnection;
import org.rosuda.REngine.Rserve.RFileInputStream;
import org.rosuda.REngine.Rserve.RFileOutputStream;
import org.rosuda.REngine.Rserve.RserveException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RIOHandler {
/**
* these data bindings do not need any pre-procesing or wrapping when loaded into an R session
*/
@SuppressWarnings("unchecked")
protected static List<Class< ? extends AbstractLiteralDataBinding>> simpleInputLiterals = Arrays.asList(LiteralByteBinding.class,
LiteralDoubleBinding.class,
LiteralFloatBinding.class,
LiteralIntBinding.class,
LiteralLongBinding.class,
LiteralShortBinding.class);
@SuppressWarnings("unchecked")
protected static List<Class< ? extends AbstractLiteralDataBinding>> simpleOutputLiterals = Arrays.asList(LiteralByteBinding.class,
LiteralDoubleBinding.class,
LiteralFloatBinding.class,
LiteralIntBinding.class,
LiteralLongBinding.class,
LiteralShortBinding.class,
LiteralStringBinding.class);
public static interface RInputFilter {
public abstract String filter(String input) throws ExceptionReport;
}
public class StringInputFilter implements RInputFilter {
public String filter(String input) throws ExceptionReport {
if (input.contains("=") || input.contains("<-"))
throw new ExceptionReport("Assignment operators found, not allowed, illegal input '" + input + "'",
ExceptionReport.INVALID_PARAMETER_VALUE);
return input;
}
}
private static Logger log = LoggerFactory.getLogger(RIOHandler.class);
private RInputFilter filter;
public RIOHandler() {
log.debug("NEW {}", this);
this.filter = new StringInputFilter();
}
public Class< ? extends IData> getInputDataType(String id, Collection<RAnnotation> annotations) {
try {
return getIODataType(RAnnotationType.INPUT, id, annotations);
}
catch (RAnnotationException e) {
String message = "Data type for id " + id + " could not be retrieved, return null";
log.error(message, e);
}
return null;
}
/**
* Searches annotations (class attribute) for Inputs / Outputs with a specific referring id
*
* @param ioType
* @param id
* @param annotations
* @return
* @throws RAnnotationException
*/
protected Class< ? extends IData> getIODataType(RAnnotationType ioType,
String id,
Collection<RAnnotation> annotations) throws RAnnotationException {
Class< ? extends IData> dataType = null;
List<RAnnotation> ioNotations = RAnnotation.filterAnnotations(annotations, ioType, RAttribute.IDENTIFIER, id);
if (ioNotations.isEmpty()) {
log.error("Missing R-script-annotation of type " + ioType.toString().toLowerCase() + " for id \"" + id
+ "\" ,datatype - class not found");
return null;
}
if (ioNotations.size() > 1) {
log.warn("R-script contains more than one annotation of type " + ioType.toString().toLowerCase()
+ " for id \"" + id + "\n" + " WPS selects the first one.");
}
RAnnotation annotation = ioNotations.get(0);
String rClass = annotation.getStringValue(RAttribute.TYPE);
dataType = RAnnotation.getDataClass(rClass);
if (dataType == null) {
log.error("R-script-annotation for " + ioType.toString().toLowerCase() + " id \"" + id
+ "\" contains unsuported data format identifier \"" + rClass + "\"");
}
return dataType;
}
public Class< ? extends IData> getOutputDataType(String id, Collection<RAnnotation> annotations) {
if (id.equalsIgnoreCase("sessionInfo") || id.equalsIgnoreCase("warnings"))
return GenericFileDataBinding.class;
try {
return getIODataType(RAnnotationType.OUTPUT, id, annotations);
}
catch (RAnnotationException e) {
String message = "Data type for id " + id + " could not be retrieved, return null";
log.error(message, e);
}
return null;
}
/**
* parses iData values to string representations which can be evaluated by Rserve, complex data will be
* preprocessed and handled here, uses parseLiteralInput for parsing literal Data
*
* @param input
* input value as databinding
* @param Rconnection
* (open)
* @return String which could be evaluated by RConnection.eval(String)
* @throws IOException
* @throws RserveException
* @throws REXPMismatchException
* @throws ExceptionReport
*/
public String parseInput(List<IData> input, RConnection connection) throws IOException,
RserveException,
REXPMismatchException,
RAnnotationException,
ExceptionReport {
String result = null;
// building an R - vector of input entries containing more than one
// value:
if (input.size() > 1) {
log.debug("Parsing input vector of length {}", input.size());
result = "c(";
// parsing elements 1..n-1 to vector:
for (int i = 0; i < input.size() - 1; i++) {
if (input.get(i).equals(null))
continue;
result += parseInput(input.subList(i, i + 1), connection);
result += ", ";
}
// parsing last element separately to vecor:
result += parseInput(input.subList(input.size() - 1, input.size()), connection);
result += ")";
}
IData ivalue = input.get(0);
log.debug("Handling input value {} with payload {}", ivalue, ivalue.getPayload());
Class< ? extends IData> iclass = ivalue.getClass();
if (ivalue instanceof ILiteralData)
return parseLiteralInput(iclass, ivalue.getPayload());
if (ivalue instanceof GenericFileDataWithGTBinding) {
GenericFileDataWithGT value = (GenericFileDataWithGT) ivalue.getPayload();
InputStream is = value.getDataStream();
String ext = value.getFileExtension();
result = streamFromWPSToRserve(connection, is, ext);
is.close();
return result;
}
if (ivalue instanceof GenericFileDataBinding) {
GenericFileData value = (GenericFileData) ivalue.getPayload();
InputStream is = value.getDataStream();
String ext = value.getFileExtension();
result = streamFromWPSToRserve(connection, is, ext);
is.close();
return result;
}
if (ivalue instanceof GTRasterDataBinding) {
GeotiffGenerator tiffGen = new GeotiffGenerator();
InputStream is = tiffGen.generateStream(ivalue, GenericFileDataConstants.MIME_TYPE_GEOTIFF, "base64");
// String ext = value.getFileExtension();
result = streamFromWPSToRserve(connection, is, "tiff");
is.close();
return result;
}
if (ivalue instanceof GTVectorDataBinding) {
GTVectorDataBinding value = (GTVectorDataBinding) ivalue;
File shp = value.getPayloadAsShpFile();
String path = shp.getAbsolutePath();
String baseName = path.substring(0, path.length() - ".shp".length());
File shx = new File(baseName + ".shx");
File dbf = new File(baseName + ".dbf");
File prj = new File(baseName + ".prj");
File shpZip = IOUtils.zip(shp, shx, dbf, prj);
InputStream is = new FileInputStream(shpZip);
String ext = "shp";
result = streamFromWPSToRserve(connection, is, ext);
is.close();
return result;
}
// if nothing was supported:
String message = "An unsuported IData Class occured for input: " + input.get(0).getClass();
log.error(message);
throw new RuntimeException(message);
}
public String parseLiteralInput(Class< ? extends IData> iClass, Object value) throws ExceptionReport {
String result = null;
if (value == null) {
log.warn("Value for is null for {} - setting it to 'NA' in R.", iClass);
result = "NA";
}
else {
String valueString = value.toString();
valueString = this.filter.filter(valueString);
if (simpleInputLiterals.contains(iClass)) {
result = valueString;
}
else if (iClass.equals(LiteralBooleanBinding.class)) {
boolean b = Boolean.parseBoolean(valueString);
if (b)
result = "TRUE";
else
result = "FALSE";
}
else if (iClass.equals(LiteralStringBinding.class)) {
result = "\"" + valueString + "\"";
}
else {
log.warn("An unsuported IData class occured for input {} with value {}. It will be interpreted as character value within R",
iClass,
valueString);
result = "\"" + valueString + "\"";
}
}
return result;
}
public IData parseOutput(RConnection connection,
String result_id,
REXP result,
Collection<RAnnotation> annotations,
RWorkspace workspace) throws IOException,
REXPMismatchException,
RserveException,
RAnnotationException,
ExceptionReport {
log.debug("parsing Output with id {} from result {}", result_id, result);
boolean wpsWorkDirIsRWorkDir = workspace.isWpsWorkDirIsRWorkDir();
String wpsWorkDir = workspace.getPath();
if (result == null) {
log.error("Result for output parsing is NULL for id {}", result_id);
throw new ExceptionReport("Result for output parsing is NULL for id " + result_id, result_id);
}
Class< ? extends IData> iClass = getOutputDataType(result_id, annotations);
log.debug("Output data type: {}", iClass.toString());
// extract mimetype from annotations (TODO: might have to be
// simplified somewhen)
List<RAnnotation> list = RAnnotation.filterAnnotations(annotations,
RAnnotationType.OUTPUT,
RAttribute.IDENTIFIER,
result_id);
if (list.size() > 1)
log.warn("Filtered for annotation by name but got more than one result! Just using the first one of : {}",
Arrays.toString(list.toArray()));
RAnnotation currentAnnotation = list.get(0);
log.debug("Current annotation: {}", currentAnnotation);
// extract filename from R
String filename = new File(result.asString()).getName();
if (iClass.equals(GenericFileDataBinding.class)) {
log.debug("Creating output with GenericFileDataBinding for file {}", filename);
String mimeType = "application/unknown";
File resultFile = new File(filename);
log.debug("Loading file " + resultFile.getAbsolutePath());
if ( !resultFile.isAbsolute())
// relative path names are relative to R work directory
resultFile = new File(connection.eval("getwd()").asString(), resultFile.getName());
if (resultFile.exists())
log.debug("Found file at {}", resultFile);
else
log.warn("Result file does not exists at {}", resultFile);
// Transfer file from R workdir to WPS workdir
File outputFile = null;
if (wpsWorkDirIsRWorkDir)
outputFile = resultFile;
else
outputFile = streamFromRserveToWPS(connection, resultFile.getAbsolutePath(), wpsWorkDir);
if ( !outputFile.exists())
throw new IOException("Output file does not exists: " + resultFile.getAbsolutePath());
String rType = currentAnnotation.getStringValue(RAttribute.TYPE);
mimeType = RDataTypeRegistry.getInstance().getType(rType).getProcessKey();
GenericFileData out = new GenericFileData(outputFile, mimeType);
return new GenericFileDataBinding(out);
}
else if (iClass.equals(GenericFileDataWithGTBinding.class)) {
log.debug("Creating output with GenericFileDataWithGTBinding for file {}", filename);
String mimeType = "application/unknown";
File resultFile = new File(filename);
log.debug("Loading file " + resultFile.getAbsolutePath());
if ( !resultFile.isAbsolute())
// relative path names are relative to R work directory
resultFile = new File(connection.eval("getwd()").asString(), resultFile.getName());
if (resultFile.exists())
log.debug("Found file at {}", resultFile);
else
log.warn("Result file does not exists at {}", resultFile);
// Transfer file from R workdir to WPS workdir
File outputFile = null;
if (wpsWorkDirIsRWorkDir)
outputFile = resultFile;
else
outputFile = streamFromRserveToWPS(connection, resultFile.getAbsolutePath(), wpsWorkDir);
if ( !outputFile.exists())
throw new IOException("Output file does not exists: " + resultFile.getAbsolutePath());
String rType = currentAnnotation.getStringValue(RAttribute.TYPE);
mimeType = RDataTypeRegistry.getInstance().getType(rType).getProcessKey();
GenericFileDataWithGT out = new GenericFileDataWithGT(outputFile, mimeType);
return new GenericFileDataWithGTBinding(out);
}
else if (iClass.equals(GTVectorDataBinding.class)) {
String mimeType = "application/unknown";
RTypeDefinition dataType = currentAnnotation.getRDataType();
File outputFile;
if (dataType.equals(RDataType.SHAPE) || dataType.equals(RDataType.SHAPE_ZIP2) && !wpsWorkDirIsRWorkDir) {
String zip = "";
REXP ev = connection.eval("zipShp(\"" + filename + "\")");
// filname = baseName (+ suffix)
String baseName = null;
if (filename.endsWith(".shp"))
baseName = filename.substring(0, filename.length() - ".shp".length());
else
baseName = filename;
// zip all -- stream --> unzip all or stream each file?
if ( !ev.isNull()) {
zip = ev.asString();
File zipfile = streamFromRserveToWPS(connection, zip, wpsWorkDir);
outputFile = IOUtils.unzip(zipfile, "shp").get(0);
}
else {
log.info("R call to zip() does not work, streaming of shapefile without zipping");
String[] dir = connection.eval("dir()").asStrings();
for (String f : dir) {
if (f.startsWith(baseName) && !f.equals(filename))
streamFromRserveToWPS(connection, f, wpsWorkDir);
}
outputFile = streamFromRserveToWPS(connection, filename, wpsWorkDir);
}
}
else {
if (wpsWorkDirIsRWorkDir) {
outputFile = new File(filename);
if ( !outputFile.isAbsolute())
// relative path names are alway relative to R work directory
outputFile = new File(connection.eval("getwd()").asString(), outputFile.getName());
}
else
outputFile = streamFromRserveToWPS(connection, filename, wpsWorkDir);
}
String rType = currentAnnotation.getStringValue(RAttribute.TYPE);
mimeType = RDataTypeRegistry.getInstance().getType(rType).getProcessKey();
GenericFileDataWithGT gfd = new GenericFileDataWithGT(outputFile, mimeType);
GTVectorDataBinding gtvec = gfd.getAsGTVectorDataBinding();
return gtvec;
}
else if (iClass.equals(GTRasterDataBinding.class)) {
String mimeType = "application/unknown";
File tempfile = streamFromRserveToWPS(connection, filename, wpsWorkDir);
String rType = currentAnnotation.getStringValue(RAttribute.TYPE);
mimeType = RDataTypeRegistry.getInstance().getType(rType).getProcessKey();
GeotiffParser tiffPar = new GeotiffParser();
FileInputStream fis = new FileInputStream(tempfile);
GTRasterDataBinding output = tiffPar.parse(fis, mimeType, "base64");
fis.close();
return output;
}
else if (iClass.equals(LiteralBooleanBinding.class)) {
log.debug("Creating output with LiteralBooleanBinding");
int tresult = result.asInteger();
switch (tresult) {
case 1:
return new LiteralBooleanBinding(true);
case 0:
return new LiteralBooleanBinding(false);
default:
break;
}
}
for (Class< ? > literal : simpleOutputLiterals) {
if (iClass.equals(literal)) {
Constructor<IData> cons = null;
try {
cons = (Constructor<IData>) iClass.getConstructors()[0];
Constructor< ? > param = cons.getParameterTypes()[0].getConstructor(String.class);
if (literal.equals(LiteralIntBinding.class)) {
// try to force conversion from R-datatype to integer
// (important for the R-data type "numeric"):
String intString = Integer.toString(result.asInteger());
return cons.newInstance(param.newInstance(intString));
}
return cons.newInstance(param.newInstance(result.asString()));
}
catch (Exception e) {
String message = "Error for parsing String to IData for " + result_id + " and class " + iClass
+ "\n" + e.getMessage();
log.error(message, e);
throw new RuntimeException(message);
}
}
}
String message = "R_Proccess: Unsuported Output Data Class declared for id " + result_id + ":" + iClass;
log.error(message);
throw new RuntimeException(message);
}
/**
* Streams a File from R workdirectory to a temporal file in the WPS4R workdirectory
* (R.Config.WORK_DIR/random folder)
*
* @param filename
* name or path of the file located in the R workdirectory
* @param wpsWorkDir
* @return Location of a file which has been streamed
* @throws IOException
* @throws FileNotFoundException
*/
private File streamFromRserveToWPS(RConnection connection, String filename, String wpsWorkDir) throws IOException,
FileNotFoundException {
File tempfile = new File(filename);
File destination = new File(wpsWorkDir);
if ( !destination.exists())
destination.mkdirs();
tempfile = new File(destination, tempfile.getName());
// Do streaming Rserve --> WPS tempfile
RFileInputStream fis = connection.openFile(filename);
FileOutputStream fos = new FileOutputStream(tempfile);
byte[] buffer = new byte[2048];
int stop = fis.read(buffer);
while (stop != -1) {
fos.write(buffer, 0, stop);
stop = fis.read(buffer);
}
fis.close();
fos.close();
// tempfile.deleteOnExit();
return tempfile;
}
/**
* Streams a file from WPS to Rserve workdirectory
*
* @param connection
* active RConnecion
* @param is
* inputstream of the inputfile
* @param ext
* basefile extension
* @return
* @throws IOException
* @throws REXPMismatchException
* @throws RserveException
*/
private String streamFromWPSToRserve(RConnection connection, InputStream is, String ext) throws IOException,
REXPMismatchException,
RserveException {
String result;
String randomname = UUID.randomUUID().toString();
String inputFileName = randomname;
// List<Class< ? extends AbstractLiteralDataBinding>> easyLiterals =
// Arrays.asList(LiteralByteBinding.class,
// LiteralDoubleBinding.class,
// LiteralFloatBinding.class,
// LiteralIntBinding.class,
// LiteralLongBinding.class,
// LiteralShortBinding.class);
RFileOutputStream rfos = connection.createFile(inputFileName);
byte[] buffer = new byte[2048];
int stop = is.read(buffer);
while (stop != -1) {
rfos.write(buffer, 0, stop);
stop = is.read(buffer);
}
rfos.flush();
rfos.close();
is.close();
// R unzips archive files and renames files with unique
// random names
// TODO: check whether input is a zip archive or not
result = connection.eval("unzipRename(" + "\"" + inputFileName + "\", " + "\"" + randomname + "\", " + "\""
+ ext + "\")").asString();
result = "\"" + result + "\"";
return result;
}
}