/** * Copyright (C) 2010 - 2016 52°North Initiative for Geospatial Open Source * Software GmbH * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. * * If the program is linked with libraries which are licensed under one of * the following licenses, the combination of the program with the linked * library is not considered a "derivative work" of the program: * * • Apache License, version 2.0 * • Apache Software License, version 1.0 * • GNU Lesser General Public License, version 3 * • Mozilla Public License, versions 1.0, 1.1 and 2.0 * • Common Development and Distribution License (CDDL), version 1.0 * * Therefore the distribution of the program linked with libraries licensed * under the aforementioned licenses, is permitted by the copyright holders * if the distribution is compliant with both the GNU General Public * License version 2 and the aforementioned licenses. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General * Public License for more details. */ package org.n52.wps.server.r; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import net.opengis.wps.x100.ProcessDescriptionType; import org.n52.wps.io.data.IData; import org.n52.wps.server.AbstractObservableAlgorithm; import org.n52.wps.server.ExceptionReport; import org.n52.wps.server.r.metadata.RAnnotationParser; import org.n52.wps.server.r.metadata.RProcessDescriptionCreator; import org.n52.wps.server.r.syntax.RAnnotation; import org.n52.wps.server.r.syntax.RAnnotationException; import org.n52.wps.server.r.syntax.RAnnotationType; import org.n52.wps.server.r.util.RExecutor; import org.n52.wps.server.r.util.RLogger; import org.n52.wps.server.r.workspace.RIOHandler; import org.n52.wps.server.r.workspace.RSessionManager; import org.n52.wps.server.r.workspace.RWorkspaceManager; import org.rosuda.REngine.REXP; import org.rosuda.REngine.REXPMismatchException; import org.rosuda.REngine.Rserve.RserveException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class GenericRProcess extends AbstractObservableAlgorithm { private static Logger log = LoggerFactory.getLogger(GenericRProcess.class); // private variables holding process information - initialization in // constructor private List<RAnnotation> annotations; private R_Config config; private List<String> errors = new ArrayList<String>(); private RExecutor executor = new RExecutor(); private RIOHandler iohandler = new RIOHandler(); private RAnnotationParser parser; private File scriptFile = null; private boolean shutdownRServerAfterRun = false; private Thread updateThread; private boolean stopUpdateThread = false; private long lastStatusUpdate = 0; public GenericRProcess(String wellKnownName) { super(wellKnownName); log.debug("NEW {}", this); } public List<String> getErrors() { return this.errors; } @Override public Class< ? extends IData> getInputDataType(String id) { return this.iohandler.getInputDataType(id, this.annotations); } @Override public Class< ? > getOutputDataType(String id) { return this.iohandler.getOutputDataType(id, this.annotations); } @Override protected ProcessDescriptionType initializeDescription() { this.config = R_Config.getInstance(); // call here because method is invoked by super constructor // Reading process information from script annotations: InputStream rScriptStream = null; try { String wkn = getWellKnownName(); log.debug("Loading file for {}", wkn); this.scriptFile = config.getScriptFileForWKN(wkn); log.debug("File loaded: {}", this.scriptFile.getAbsolutePath()); log.info("Initializing description for {}", this.toString()); if (this.scriptFile == null) { log.warn("Loaded script file is {}", this.scriptFile); throw new ExceptionReport("Cannot create process description because R script fill is null", ExceptionReport.NO_APPLICABLE_CODE); } rScriptStream = new FileInputStream(this.scriptFile); if (this.parser == null) this.parser = new RAnnotationParser(this.config); // prevent NullpointerException this.annotations = this.parser.parseAnnotationsfromScript(rScriptStream); // submits annotation with process informations to // ProcessdescriptionCreator: RProcessDescriptionCreator creator = new RProcessDescriptionCreator(this.config); ProcessDescriptionType doc = creator.createDescribeProcessType(this.annotations, wkn, config.getScriptURL(wkn), config.getSessionInfoURL()); log.debug("Created process description for {}:\n{}", wkn, doc.xmlText()); return doc; } catch (RAnnotationException rae) { log.error(rae.getMessage()); throw new RuntimeException("Annotation error while parsing process description: " + rae.getMessage(), rae); } catch (IOException ioe) { log.error("I/O error while parsing process description: " + ioe.getMessage()); throw new RuntimeException("I/O error while parsing process description: " + ioe.getMessage(), ioe); } catch (ExceptionReport e) { log.error(e.getMessage(), e); throw new RuntimeException("Error creating process description: " + e.getMessage(), e); } finally { try { if (rScriptStream != null) rScriptStream.close(); } catch (IOException e) { log.error("Error closing script stream.", e); } } } public Map<String, IData> run(Map<String, List<IData>> inputData) throws ExceptionReport { log.info("Running {} \n\tInput data: {}", this.toString(), Arrays.toString(inputData.entrySet().toArray())); FilteredRConnection rCon = null; try { rCon = config.openRConnection(); RLogger.logGenericRProcess(rCon, "Running algorithm with input " + Arrays.deepToString(inputData.entrySet().toArray())); RSessionManager session = new RSessionManager(rCon, config); session.configureSession(getWellKnownName(), executor); RWorkspaceManager workspace = new RWorkspaceManager(rCon, this.iohandler, config); String originalWorkDir = workspace.prepareWorkspace(inputData, getWellKnownName()); List<RAnnotation> resAnnotList = RAnnotation.filterAnnotations(this.annotations, RAnnotationType.RESOURCE); workspace.loadResources(resAnnotList); List<RAnnotation> inAnnotations = RAnnotation.filterAnnotations(this.annotations, RAnnotationType.INPUT); workspace.loadInputValues(inputData, inAnnotations); if (log.isDebugEnabled()) workspace.saveImage("preExecution"); File scriptFile = config.getScriptFileForWKN(getWellKnownName()); //add simple status logging functionality //log status via updateStatus-method to temp file that is created in GenericRProcess class //temp file is read by process if changed and the contents are to the update-method of the ISubject interface String script = "tmpStatusFile <- tempfile()"; rCon.voidEval(script); File tmpStatusFile; REXP tmpStatusFileREXP = rCon.eval("tmpStatusFile"); if(tmpStatusFileREXP.isString()){ try { tmpStatusFile = new File(tmpStatusFileREXP.asString()); StringBuilder statusScriptString = new StringBuilder(); statusScriptString.append("writelock = function() {\n "); statusScriptString.append("file.create(paste0(tmpStatusFile, \"" + R_Config.LOCK_SUFFIX + "\"))\n"); statusScriptString.append("}\n "); statusScriptString.append("removelock = function(i) {\n "); statusScriptString.append("file.remove(paste0(tmpStatusFile, \"" + R_Config.LOCK_SUFFIX + "\"))\n"); statusScriptString.append("}\n "); statusScriptString.append("updateStatus = function(i) {\n "); statusScriptString.append("writelock()\n "); statusScriptString.append("write(as.character(i),file=tmpStatusFile,append=F)\n"); statusScriptString.append("removelock()\n"); statusScriptString.append("}\n "); rCon.voidEval(statusScriptString.toString()); tmpStatusFile.createNewFile(); startUpdateListener(tmpStatusFile); } catch (REXPMismatchException e) { log.debug("Could not parse String generated by R method tempfile() to Java File. No status updates are possible.", e); } } HashMap<String, IData> result = null; boolean success = executor.executeScript(scriptFile, rCon); if (success) { List<RAnnotation> outAnnotations = RAnnotation.filterAnnotations(this.annotations, RAnnotationType.OUTPUT); result = workspace.saveOutputValues(outAnnotations); result = session.saveInfos(result); } else { String msg = "Failure while executing R script. See logs for details"; log.error(msg); throw new ExceptionReport(msg, getClass().getName()); } if (log.isDebugEnabled()) workspace.saveImage("afterExecution"); log.debug("RESULT: " + Arrays.toString(result.entrySet().toArray())); session.cleanUp(); workspace.cleanUpInR(originalWorkDir); workspace.cleanUpWithWPS(); return result; } catch (IOException e) { String message = "Attempt to run R script file failed:\n" + e.getClass() + " - " + e.getLocalizedMessage() + "\n" + e.getCause(); log.error(message, e); throw new ExceptionReport(message, e.getClass().getName(), e); } catch (RAnnotationException e) { String message = "R script cannot be executed due to invalid annotations."; log.error(message, e); throw new ExceptionReport(message, e.getClass().getName(), e); } catch (RserveException e) { log.error("Rserve problem executing script: " + e.getMessage(), e); throw new ExceptionReport("Rserve problem executing script: " + e.getMessage(), "R", ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } catch (REXPMismatchException e) { String message = "An R Parsing Error occoured:\n" + e.getMessage() + " - " + e.getClass() + " - " + e.getLocalizedMessage() + "\n" + e.getCause(); log.error(message, e); throw new ExceptionReport(message, "R", "R_Connection", e); } finally { if (rCon != null) { if (shutdownRServerAfterRun) { log.debug("Shutting down R completely..."); try { rCon.serverShutdown(); } catch (RserveException e) { String message = "Error during R server shutdown:\n" + e.getMessage() + " - " + e.getClass() + " - " + e.getLocalizedMessage() + "\n" + e.getCause(); log.error(message, e); throw new ExceptionReport(message, "R", "R_Connection", e); } } else rCon.close(); } if(updateThread != null && updateThread.isAlive()){ stopUpdateThread = true; } } } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("GenericRProcess [script = "); sb.append(this.scriptFile); if (this.annotations != null) { sb.append(", annotations = "); sb.append(Arrays.toString(this.annotations.toArray())); } sb.append("]"); return sb.toString(); } private void startUpdateListener(final File tmpStatusFile){ updateThread = new Thread("WPS4R-update-thread"){ @Override public void run() { while(true){ if(stopUpdateThread){ break; } try { sleep(1000); } catch (InterruptedException e) { log.error("InterruptedException while trying to sleep WPS4R-update-thread.", e); } //lock exists continue if(new File(tmpStatusFile.getAbsolutePath().concat(R_Config.LOCK_SUFFIX)).exists()){ continue; } try { String updateMessage = readTmpStatusFile(tmpStatusFile); if(updateMessage == null || updateMessage.isEmpty()){ continue; } //try parsing status as integer and update process status if successful try{ Integer percentage = Integer.parseInt(updateMessage.trim()); update(percentage); }catch(NumberFormatException e){ log.info("Status could not be parsed to integer: " + updateMessage); //update status with message (works only for WPS 1.0) update(updateMessage); } } catch (IOException e) { log.error("Could not read status from file: " + tmpStatusFile.getAbsolutePath(), e); } } } }; updateThread.start(); } private String readTmpStatusFile(File tmpStatusFile) throws IOException{ String content = ""; long statusFileModified = tmpStatusFile.lastModified(); log.debug("File modified: " + (statusFileModified > lastStatusUpdate)); if(lastStatusUpdate == 0 || statusFileModified > lastStatusUpdate){ BufferedReader bufferedReader = new BufferedReader(new FileReader(tmpStatusFile)); String line = ""; while((line = bufferedReader.readLine()) != null){ content = content.concat(line + "\n"); } bufferedReader.close(); lastStatusUpdate = statusFileModified; } return content; } }