/** * Copyright (C) 2010 - 2016 52°North Initiative for Geospatial Open Source * Software GmbH * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. * * If the program is linked with libraries which are licensed under one of * the following licenses, the combination of the program with the linked * library is not considered a "derivative work" of the program: * * • Apache License, version 2.0 * • Apache Software License, version 1.0 * • GNU Lesser General Public License, version 3 * • Mozilla Public License, versions 1.0, 1.1 and 2.0 * • Common Development and Distribution License (CDDL), version 1.0 * * Therefore the distribution of the program linked with libraries licensed * under the aforementioned licenses, is permitted by the copyright holders * if the distribution is compliant with both the GNU General Public * License version 2 and the aforementioned licenses. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General * Public License for more details. */ package org.n52.wps.server.r.workspace; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import org.n52.wps.io.data.IData; import org.n52.wps.server.ExceptionReport; import org.n52.wps.server.r.FilteredRConnection; import org.n52.wps.server.r.RWPSConfigVariables; import org.n52.wps.server.r.RWPSSessionVariables; import org.n52.wps.server.r.R_Config; import org.n52.wps.server.r.data.R_Resource; import org.n52.wps.server.r.syntax.RAnnotation; import org.n52.wps.server.r.syntax.RAnnotationException; import org.n52.wps.server.r.syntax.RAttribute; import org.n52.wps.server.r.util.RExecutor; import org.n52.wps.server.r.util.RLogger; import org.rosuda.REngine.REXP; import org.rosuda.REngine.REXPMismatchException; import org.rosuda.REngine.REngine; import org.rosuda.REngine.Rserve.RserveException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * * @author Daniel Nüst * */ public class RWorkspaceManager { private static Logger log = LoggerFactory.getLogger(RWorkspaceManager.class); private static final String RDATA_FILE_EXTENSION = "RData"; private R_Config config; private FilteredRConnection connection; /** * Indicates if the WPS working directory should be deleted after process execution */ private boolean deleteWPSWorkDirectory = true; private RExecutor executor; private RIOHandler iohandler; private RWorkspace workspace; public RWorkspaceManager(FilteredRConnection connection, RIOHandler iohandler, R_Config config) { this.connection = connection; this.workspace = new RWorkspace(); this.executor = new RExecutor(); this.iohandler = iohandler; this.config = config; log.debug("NEW {}", this); } /** * * @param originalWorkDir * the working directory of R after the clean up is finished */ public void cleanUpInR(String originalWorkDir) { log.debug("Cleaning up workspace from R ..."); // R_Config config = R_Config.getInstance(); // RConnection connection = rCon; // if (rCon == null || !rCon.isConnected()) { // log.debug("[R] opening new connection for cleanup..."); // connection = config.openRConnection(); // } RLogger.log(connection, "Workspace after process run:"); RLogger.logWorkspaceContent(this.connection); log.debug("Deleting work directory {}", originalWorkDir); boolean b = this.workspace.deleteCurrentAndSetWorkdir(this.connection, originalWorkDir); if ( !b) log.debug("Could not delete workdir (completely) with R, remaining files: {}", this.workspace.listFiles()); } public void cleanUpWithWPS() { log.debug("Cleaning up workspace from Java ..."); try { if (this.deleteWPSWorkDirectory) { // try to delete current local workdir - folder File workdir = new File(workspace.getPath()); if ( !workdir.exists()) return; boolean deleted = deleteRecursive(workdir); if ( !deleted) log.warn("Failed to delete temporary WPS Workdirectory '{}', remaining files: {}", workdir.getAbsolutePath(), this.workspace.listFiles()); } } catch (Exception e) { log.error("Problem deleting the wps work directory.", e); } } /** * Deletes File or Directory completely with its content * * @param in * File or directory * @return true if all content could be deleted */ private boolean deleteRecursive(File in) { boolean success = true; if ( !in.exists()) { return false; } if (in.isDirectory()) { File[] files = in.listFiles(); for (File file : files) { if (file.isFile()) { success = success && file.delete(); } if (file.isDirectory()) { success = success && deleteRecursive(file); } } } if (success) { success = success && in.delete(); } return success; } public void loadInputValues(Map<String, List<IData>> inputData, List<RAnnotation> inAnnotations) throws RAnnotationException, ExceptionReport { log.debug("Loading input values..."); // Searching for missing inputs to apply standard values: log.debug("in annonations: " + Arrays.toString(inAnnotations.toArray())); // ------------------------------- // Input value initialization: // ------------------------------- HashMap<String, String> inputValues = new HashMap<String, String>(); ArrayList<String> inputValuesWithValues = new ArrayList<String>(); for (Entry<String, List<IData>> entry : inputData.entrySet()) { // parses input values to R-compatible literals and streams input files to workspace try { String entryRValue = this.iohandler.parseInput(entry.getValue(), connection); log.debug("Parsed input for '{}' to '{}' based on value '{}'", entry.getKey(), entryRValue, entry.getValue()); inputValues.put(entry.getKey(), entryRValue); inputValuesWithValues.add(entry.getKey()); } catch (RserveException e) { log.error("Error parsing input value {}", entry, e); throw new ExceptionReport("Error parsing input value: " + entry, ExceptionReport.INVALID_PARAMETER_VALUE, e); } catch (REXPMismatchException e) { log.error("Error parsing input value {}", entry, e); throw new ExceptionReport("Error parsing input value: " + entry, ExceptionReport.INVALID_PARAMETER_VALUE, e); } catch (IOException e) { log.error("Error parsing input value {}", entry, e); throw new ExceptionReport("Error parsing input value: " + entry, ExceptionReport.INVALID_PARAMETER_VALUE, e); } } log.debug("Input: {}", Arrays.toString(inAnnotations.toArray())); // parses default values to R-compatible literals if no value has been set for (RAnnotation rAnnotation : inAnnotations) { String id = rAnnotation.getStringValue(RAttribute.IDENTIFIER); if ( !inputValuesWithValues.contains(id)) { String value = rAnnotation.getStringValue(RAttribute.DEFAULT_VALUE); Class< ? extends IData> iClass = this.iohandler.getInputDataType(id, inAnnotations); String inputValue = this.iohandler.parseLiteralInput(iClass, value); log.debug("Loaded default input value '{}' for '{}'", inputValue, rAnnotation); inputValues.put(id, inputValue); } } log.debug("Assigns (including defaults): {}", Arrays.toString(inputValues.entrySet().toArray())); // assign values to the (clean) workspace: log.debug("Assigning values..."); Set<Entry<String, String>> inputValues2 = inputValues.entrySet(); for (Entry<String, String> entry : inputValues2) { // use eval, not assign (assign only parses strings) String statement = entry.getKey() + " <- " + entry.getValue(); log.debug("Running statement '{}'", statement); try { connection.filteredEval(statement); } catch (RserveException e) { log.error("Error executing statement '{}'", statement, e); throw new ExceptionReport("Error executing statement: " + statement + ": " + e.getMessage(), ExceptionReport.INVALID_PARAMETER_VALUE, e); } } RLogger.log(connection, "Session after loading input values:"); RLogger.logSessionContent(connection); } public void loadResources(List<RAnnotation> resources) throws RAnnotationException, ExceptionReport, IOException { try { loadResourcesListInSession(resources); } catch (RserveException e) { log.error("Problem loading resources list to session, list: {}", resources, e); } loadResourcesToWorkspace(resources); log.debug("Workspace contents after resource loading: {}", this.workspace.listFiles()); RLogger.log(connection, "Session after resource loading:"); RLogger.logSessionContent(connection); RLogger.log(connection, "Workspace after resource loading:"); RLogger.logWorkspaceContent(connection); } private void loadResourcesListInSession(Collection<RAnnotation> resources) throws RserveException, RAnnotationException { log.debug("Saving resources in session: {}", resources); String wpsScriptResources = null; // Assign and concatenate lists of resources given by the ressource annotations wpsScriptResources = "list()"; connection.filteredEval(RWPSSessionVariables.SCRIPT_RESOURCES + " <- " + wpsScriptResources); for (RAnnotation annotation : resources) { wpsScriptResources = annotation.getStringValue(RAttribute.NAMED_LIST_R_SYNTAX); connection.filteredEval(RWPSSessionVariables.SCRIPT_RESOURCES + " <- " + "append(" + RWPSSessionVariables.SCRIPT_RESOURCES + ", " + wpsScriptResources + ")"); } log.debug("Assigned recource urls to variable '{}': {}", RWPSSessionVariables.SCRIPT_RESOURCES, wpsScriptResources); RLogger.logVariable(connection, RWPSSessionVariables.SCRIPT_RESOURCES); } private void loadResourcesToWorkspace(Collection<RAnnotation> resources) throws RAnnotationException, ExceptionReport, IOException { log.debug("Loading resources into workspace: {}", resources); for (RAnnotation resourceAnnotation : resources) { Object resObject = resourceAnnotation.getObjectValue(RAttribute.NAMED_LIST); Collection< ? > resourceCollection; if (resObject instanceof Collection< ? >) resourceCollection = (Collection< ? >) resObject; else { log.warn("Unsupported resource object: {}", resObject); continue; } for (Object element : resourceCollection) { R_Resource resource; if (element instanceof R_Resource) resource = (R_Resource) element; else { log.warn("Unsupported resource element: {}", element); continue; } File resourceFile = resource.getFullResourcePath(this.config); if (resourceFile == null || !resourceFile.exists()) { throw new ExceptionReport("Resource does not exist: " + resourceAnnotation, ExceptionReport.NO_APPLICABLE_CODE); } log.debug("Loading resource {} from file {} (directory: {})", resource, resourceFile, resourceFile.isDirectory()); streamFromWPSToRserve(resourceFile); } } log.debug("Loaded resources, workspace files: {}", this.workspace.listFiles()); } /** * @return the original work directory or the R session */ public String prepareWorkspace(Map<String, List<IData>> inputData, String processWKN) throws RserveException, REXPMismatchException, ExceptionReport, FileNotFoundException, IOException, RAnnotationException { log.debug("Preparing workspace..."); log.debug("Rengine: {} | R server version: {}", REngine.getLastEngine(), connection.getServerVersion()); // Retrieve the preset R working directory (R will be reset to // this directory after the process run) String originalWD = connection.eval("getwd()").asString(); // Set R working directory according to configuration String strategy = this.config.getConfigVariable(RWPSConfigVariables.R_WORK_DIR_STRATEGY); boolean isRserveOnLocalhost = this.config.getRServeHost().equalsIgnoreCase("localhost"); String workDirNameSetting = null; try { workDirNameSetting = this.config.getConfigVariableFullPath(RWPSConfigVariables.R_WORK_DIR_NAME); } catch (ExceptionReport e) { log.error("The config variable {} references a non-existing directory. This will be an issue if the variable is used. The current strategy is '{}'.", RWPSConfigVariables.R_WORK_DIR_NAME, strategy, e); } this.workspace.setWorkingDirectory(this.connection, originalWD, strategy, isRserveOnLocalhost, workDirNameSetting); return originalWD; } /** * saves an image to the working directory that may help debugging R scripts */ public boolean saveImage(String name) { String filename = name + "." + RDATA_FILE_EXTENSION; try { REXP result = connection.eval("save.image(file=\"" + filename + "\")"); log.debug("Saved image to {} with result {}", filename, result); return true; } catch (RserveException e) { log.error("Could not save image to {}", filename, e); return false; } } /** * @return the result has including sessionInfo() and warnings() * @throws REXPMismatchException */ public HashMap<String, IData> saveOutputValues(Collection<RAnnotation> outAnnotations) throws RAnnotationException, ExceptionReport { HashMap<String, IData> result = new HashMap<String, IData>(); for (RAnnotation rAnnotation : outAnnotations) { String resultId = rAnnotation.getStringValue(RAttribute.IDENTIFIER); REXP evalResult; try { evalResult = connection.eval(resultId); } catch (RserveException e) { log.error("Could not find value for annotation {} in the current session, result id: {}", rAnnotation, resultId, e); throw new ExceptionReport("Error saving output value " + resultId, ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } // TODO depending on the generated outputs deleteWorkDirectory must be set! try { IData output = this.iohandler.parseOutput(connection, resultId, evalResult, outAnnotations, this.workspace); result.put(resultId, output); log.debug("Output for {} is {} with payload {}", resultId, output, output.getPayload()); } catch (ExceptionReport e) { throw e; } catch (RserveException e) { log.error("Could not create output for {}", resultId, e); } catch (IOException e) { log.error("Could not create output for {}", resultId, e); } catch (REXPMismatchException e) { log.error("Could not create output for {}", resultId, e); } } return result; } private void streamFromWPSToRserve(File source) throws IOException { streamFromWPSToRserve(source, RWorkspace.ROOT); } private void streamFromWPSToRserve(File source, String path) throws IOException { StringBuilder sb = new StringBuilder(); sb.append(path); sb.append("/"); sb.append(source.getName()); String name = sb.toString(); log.debug("Copying {} (directory: {}, path: '{}') to as '{}' to {} ", source, source.isDirectory(), path, name, this.workspace); if ( !source.isDirectory()) { this.workspace.copyFile(source, name, connection); } else { // create directory and append path for recursive calls try { // create subdir in R this.workspace.createDirectory(name, this.connection); String[] files = source.list(); for (String file : files) { File sourceFile = new File(source, file); streamFromWPSToRserve(sourceFile, name); } } catch (RserveException e) { log.error("Error creating directory in workdir", e); throw new IOException(e); } } } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("RWorkspaceManager ["); if (connection != null) { builder.append("connection="); builder.append(connection); builder.append(", "); } builder.append("deleteWPSWorkDirectory="); builder.append(deleteWPSWorkDirectory); builder.append(", "); if (executor != null) { builder.append("executor="); builder.append(executor); builder.append(", "); } if (iohandler != null) { builder.append("iohandler="); builder.append(iohandler); builder.append(", "); } if (workspace != null) { builder.append("workspace="); builder.append(workspace); } builder.append("]"); return builder.toString(); } }