ExecuteServiceImpl.java

/**
 * Licensed to The Apereo Foundation under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 *
 * The Apereo Foundation licenses this file to you under the Educational
 * Community License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License
 * at:
 *
 *   http://opensource.org/licenses/ecl2.txt
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 */

package org.opencastproject.execute.impl;

import org.opencastproject.execute.api.ExecuteException;
import org.opencastproject.execute.api.ExecuteService;
import org.opencastproject.job.api.AbstractJobProducer;
import org.opencastproject.job.api.Job;
import org.opencastproject.mediapackage.MediaPackage;
import org.opencastproject.mediapackage.MediaPackageElement;
import org.opencastproject.mediapackage.MediaPackageElement.Type;
import org.opencastproject.mediapackage.MediaPackageElementBuilderFactory;
import org.opencastproject.mediapackage.MediaPackageElementFlavor;
import org.opencastproject.mediapackage.MediaPackageElementParser;
import org.opencastproject.mediapackage.MediaPackageException;
import org.opencastproject.mediapackage.MediaPackageParser;
import org.opencastproject.mediapackage.UnsupportedElementException;
import org.opencastproject.security.api.OrganizationDirectoryService;
import org.opencastproject.security.api.SecurityService;
import org.opencastproject.security.api.UserDirectoryService;
import org.opencastproject.serviceregistry.api.ServiceRegistry;
import org.opencastproject.serviceregistry.api.ServiceRegistryException;
import org.opencastproject.util.ConfigurationException;
import org.opencastproject.util.IoSupport;
import org.opencastproject.util.LoadUtil;
import org.opencastproject.util.NotFoundException;
import org.opencastproject.workspace.api.Workspace;

import org.apache.commons.lang3.StringUtils;
import org.osgi.framework.BundleContext;
import org.osgi.service.cm.ManagedService;
import org.osgi.service.component.ComponentContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Dictionary;
import java.util.HashSet;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Scanner;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Implements a service that runs CLI commands with MediaPackage elements as arguments
 */
public class ExecuteServiceImpl extends AbstractJobProducer implements ExecuteService, ManagedService  {

  public enum Operation {
    Execute_Element, Execute_Mediapackage
  }

  /** The logging facility */
  private static final Logger logger = LoggerFactory.getLogger(ExecuteServiceImpl.class);

  /** Reference to the receipt service */
  private ServiceRegistry serviceRegistry = null;

  /** The security service */
  protected SecurityService securityService = null;

  /** The user directory service */
  protected UserDirectoryService userDirectoryService = null;

  /** The organization directory service */
  protected OrganizationDirectoryService organizationDirectoryService = null;

  /** The workspace service */
  protected Workspace workspace;

  /**
   * List of allowed commands that can be run with an executor. By convention, an empty set doesn't mean any command can be run.
   * An '*'  in the service configuration means any command can be executed
   */
  protected final Set<String> allowedCommands = new HashSet<String>();

  /** Bundle property specifying which commands can be run with this executor */
  public static final String COMMANDS_ALLOWED_PROPERTY = "commands.allowed";

  /** The collection for the executor files */
  public static final String COLLECTION = "executor";

  /** To allow command-line parameter substitutions configured globally i.e. in config.properties */
  private BundleContext bundleContext;

  /** To allow command-line parameter substitutions configured at the service level */
  @SuppressWarnings("rawtypes")
  private Dictionary properties = null;

  /** The approximate load placed on the system by running an execute operation */
  public static final float DEFAULT_EXECUTE_JOB_LOAD = 1.0f;

  /** The key to look for in the service configuration file to override the {@link DEFAULT_EXECUTE_JOB_LOAD} */
  public static final String EXECUTE_JOB_LOAD_KEY = "job.load.execute";

  private float executeJobLoad = 1.0f;

  /**
   * Creates a new instance of the execute service.
   */
  public ExecuteServiceImpl() {
    super(JOB_TYPE);
  }

  /**
   * Activates this component with its properties once all of the collaborating services have been set
   * 
   * @param cc
   *          The component's context, containing the properties used for configuration
   */
  @Override
  public void activate(ComponentContext cc) {
    super.activate(cc);

    properties = cc.getProperties();

    if (properties != null) {
      String commandString = (String) properties.get(COMMANDS_ALLOWED_PROPERTY);
      if (commandString != null)
        for (String command : commandString.split("\\s+"))
          allowedCommands.add(command);
    }

    this.bundleContext = cc.getBundleContext();
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement)
   */
  @Override
  public Job execute(String exec, String params, MediaPackageElement inElement) throws ExecuteException {
    return execute(exec, params, inElement, null, null, executeJobLoad);
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement,
   *                                                             float)
   */
  @Override
  public Job execute(String exec, String params, MediaPackageElement inElement, float load) throws ExecuteException {
    return execute(exec, params, inElement, null, null, load);
  }


  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement,
   *                                                             java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement.Type)
   */
  @Override
  public Job execute(String exec, String params, MediaPackageElement inElement, String outFileName, Type expectedType)
          throws ExecuteException, IllegalArgumentException {
    return execute(exec, params, inElement, null, null, executeJobLoad);
  }
  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement,
   *                                                             java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement.Type,
   *                                                             float)
   * @throws IllegalArgumentException
   *           if the input arguments are incorrect
   * @throws ExecuteException
   *           if an internal error occurs
   */
  @Override
  public Job execute(String exec, String params, MediaPackageElement inElement, String outFileName, Type expectedType, float load)
          throws ExecuteException, IllegalArgumentException {

    logger.debug("Creating Execute Job for command: {}", exec);

    if (StringUtils.isBlank(exec))
      throw new IllegalArgumentException("The command to execute cannot be null");

    if (StringUtils.isBlank(params))
      throw new IllegalArgumentException("The command arguments cannot be null");

    if (inElement == null)
      throw new IllegalArgumentException("The input MediaPackage element cannot be null");

    outFileName = StringUtils.trimToNull(outFileName);
    if ((outFileName == null) && (expectedType != null) || (outFileName != null) && (expectedType == null))
      throw new IllegalArgumentException("Expected element type and output filename cannot be null");

    try {
      List<String> paramList = new ArrayList<String>(5);
      paramList.add(exec);
      paramList.add(params);
      paramList.add(MediaPackageElementParser.getAsXml(inElement));
      paramList.add(outFileName);
      paramList.add((expectedType == null) ? null : expectedType.toString());

      return serviceRegistry.createJob(JOB_TYPE, Operation.Execute_Element.toString(), paramList, load);

    } catch (ServiceRegistryException e) {
      throw new ExecuteException(String.format("Unable to create a job of type '%s'", JOB_TYPE), e);
    } catch (MediaPackageException e) {
      throw new ExecuteException("Error serializing an element", e);
    }
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackage,
   *                                                             java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement.Type)
   */
  @Override
  public Job execute(String exec, String params, MediaPackage mp, String outFileName, Type expectedType)
          throws ExecuteException {
    return execute(exec, params, mp, outFileName, expectedType, 1.0f);
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.execute.api.ExecuteService#execute(java.lang.String, java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackage,
   *                                                             java.lang.String,
   *                                                             org.opencastproject.mediapackage.MediaPackageElement.Type,
   *                                                             float)
   */
  @Override
  public Job execute(String exec, String params, MediaPackage mp, String outFileName, Type expectedType, float load)
          throws ExecuteException {
    if (StringUtils.isBlank(exec))
      throw new IllegalArgumentException("The command to execute cannot be null");

    if (StringUtils.isBlank(params))
      throw new IllegalArgumentException("The command arguments cannot be null");

    if (mp == null)
      throw new IllegalArgumentException("The input MediaPackage cannot be null");

    outFileName = StringUtils.trimToNull(outFileName);
    if ((outFileName == null) && (expectedType != null) || (outFileName != null) && (expectedType == null))
      throw new IllegalArgumentException("Expected element type and output filename cannot be null");

    try {
      List<String> paramList = new ArrayList<String>(5);
      paramList.add(exec);
      paramList.add(params);
      paramList.add(MediaPackageParser.getAsXml(mp));
      paramList.add(outFileName);
      paramList.add((expectedType == null) ? null : expectedType.toString());

      return serviceRegistry.createJob(JOB_TYPE, Operation.Execute_Mediapackage.toString(), paramList, load);
    } catch (ServiceRegistryException e) {
      throw new ExecuteException(String.format("Unable to create a job of type '%s'", JOB_TYPE), e);
    }
  }

  /**
   * {@inheritDoc}
   *
   * @throws ExecuteException
   *
   * @see org.opencastproject.job.api.AbstractJobProducer#process(org.opencastproject.job.api.Job)
   */
  @Override
  protected String process(Job job) throws ExecuteException {
    List<String> arguments = new ArrayList<String>(job.getArguments());

    // Check this operation is allowed
    if (!allowedCommands.contains("*") && !allowedCommands.contains(arguments.get(0)))
      throw new ExecuteException("Command '" + arguments.get(0) + "' is not allowed");

    String outFileName = null;
    String strAux = null;
    MediaPackage mp = null;
    Type expectedType = null;
    MediaPackageElement element = null;
    Operation op = null;

    try {
      op = Operation.valueOf(job.getOperation());

      int nargs = arguments.size();

      if (nargs != 3 && nargs != 5) {
        throw new IndexOutOfBoundsException("Incorrect number of parameters for operation execute_" + op + ": "
                + arguments.size());
      }
      if (nargs == 5) {
        strAux = arguments.remove(4);
        expectedType = (strAux == null) ? null : Type.valueOf(strAux);
        outFileName = StringUtils.trimToNull(arguments.remove(3));
        if ((StringUtils.isNotBlank(outFileName) && (expectedType == null))
            || (StringUtils.isBlank(outFileName) && (expectedType != null))) {
          throw new ExecuteException("The output type and filename must be both specified");
        }
        outFileName = (outFileName == null) ? null : job.getId() + "_" + outFileName;
      }

      switch (op) {
        case Execute_Mediapackage:
          mp = MediaPackageParser.getFromXml(arguments.remove(2));
          return doProcess(arguments, mp, outFileName, expectedType);
        case Execute_Element:
          element = MediaPackageElementParser.getFromXml(arguments.remove(2));
          return doProcess(arguments, element, outFileName, expectedType);
        default:
          throw new IllegalStateException("Don't know how to handle operation '" + job.getOperation() + "'");
      }

    } catch (MediaPackageException e) {
      throw new ExecuteException("Error unmarshalling the input mediapackage/element", e);
    } catch (IllegalArgumentException e) {
      throw new ExecuteException("This service can't handle operations of type '" + op + "'", e);
    } catch (IndexOutOfBoundsException e) {
      throw new ExecuteException("The argument list for operation '" + op + "' does not meet expectations", e);
    }
  }


  /**
   * Does the actual processing
   *
   * @param arguments
   *          The list containing the program and its arguments
   * @param mp
   *          MediaPackage used in the operation
   * @param outFileName
   *          The name of the resulting file
   * @param expectedType
   *          The expected element type
   * @return A {@code String} containing the command output
   * @throws ExecuteException
   *           if some internal error occurred
   */
  protected String doProcess(List<String> arguments, MediaPackage mp, String outFileName, Type expectedType)
          throws ExecuteException {

    String params = arguments.remove(1);

    File outFile = null;
    MediaPackageElement[] elementsByFlavor = null;

    try {
      if (outFileName != null) {
        // FIXME : Find a better way to place the output File
        File firstElement = workspace.get(mp.getElements()[0].getURI());
        outFile = new File(firstElement.getParentFile(), outFileName);
      }

      // Get the substitution pattern.
      // The following pattern matches, any construct with the form
      // #{name}
      // , where 'name' is the value of a certain property. It is stored in the backreference group 1.
      // Optionally, expressions can take a parameter, like
      // #{name(parameter)}
      // , where 'parameter' is the name of a certain parameter.
      // If specified, 'parameter' is stored in the group 2. Otherwise it's null.
      // Both name and parameter match any character sequence that does not contain {, }, ( or ) .
      Pattern pat = Pattern.compile("#\\{([^\\{\\}\\(\\)]+)(?:\\(([^\\{\\}\\(\\)]+)\\))?\\}");

      // Substitute the appearances of the patterns with the actual absolute paths
      Matcher matcher = pat.matcher(params);
      StringBuffer sb = new StringBuffer();
      while (matcher.find()) {
        // group(1) = property. group(2) = (optional) parameter
        if (matcher.group(1).equals("id")) {
          matcher.appendReplacement(sb, mp.getIdentifier().toString());
        } else if (matcher.group(1).equals("flavor")) {
          elementsByFlavor = mp.getElementsByFlavor(MediaPackageElementFlavor.parseFlavor(matcher.group(2)));
          if (elementsByFlavor.length == 0)
            throw new ExecuteException("No elements in the MediaPackage match the flavor '" + matcher.group(1) + "'.");

          if (elementsByFlavor.length > 1)
            logger.warn("Found more than one element with flavor '{}'. Using {} by default...", matcher.group(1),
                    elementsByFlavor[0].getIdentifier());

          File elementFile = workspace.get(elementsByFlavor[0].getURI());
          matcher.appendReplacement(sb, elementFile.getAbsolutePath());
        } else if (matcher.group(1).equals("out")) {
          matcher.appendReplacement(sb, outFile.getAbsolutePath());
        } else if (properties.get(matcher.group(1)) != null) {
          matcher.appendReplacement(sb, (String) properties.get(matcher.group(1)));
        } else if (bundleContext.getProperty(matcher.group(1)) != null) {
          matcher.appendReplacement(sb, bundleContext.getProperty(matcher.group(1)));
        }
      }
      matcher.appendTail(sb);
      params = sb.toString();
    } catch (IllegalArgumentException e) {
      throw new ExecuteException("Tag 'flavor' must specify a valid MediaPackage element flavor.", e);
    } catch (NotFoundException e) {
      throw new ExecuteException("The element '" + elementsByFlavor[0].getURI().toString()
              + "' does not exist in the workspace.", e);
    } catch (IOException e) {
      throw new ExecuteException("Error retrieving MediaPackage element from workspace: '"
              + elementsByFlavor[0].getURI().toString() + "'.", e);
    }

    arguments.addAll(splitParameters(params));

    return runCommand(arguments, outFile, expectedType);
  }

  /**
   * Does the actual processing
   *
   * @param arguments
   *          The list containing the program and its arguments
   * @param outFileName
   *          The name of the resulting file
   * @param expectedType
   *          The expected element type
   * @return A {@code String} containing the command output
   * @throws ExecuteException
   *           if some internal error occurred
   */
  protected String doProcess(List<String> arguments, MediaPackageElement element, String outFileName, Type expectedType)
          throws ExecuteException {

    // arguments(1) contains a list of space-separated arguments for the command
    String params = arguments.remove(1);
    arguments.addAll(splitParameters(params));

    File outFile = null;

    try {
      // Get the track file from the workspace
      File trackFile = workspace.get(element.getURI());

      // Put the destination file in the same folder as the source file
      if (outFileName != null)
        outFile = new File(trackFile.getParentFile(), outFileName);

      // Substitute the appearances of the patterns with the actual absolute paths
      for (int i = 1; i < arguments.size(); i++) {
        if (arguments.get(i).contains(INPUT_FILE_PATTERN)) {
          arguments.set(i, arguments.get(i).replace(INPUT_FILE_PATTERN, trackFile.getAbsolutePath()));
          continue;
        }
        if (arguments.get(i).contains(OUTPUT_FILE_PATTERN)) {
          if (outFile != null) {
            arguments.set(i, arguments.get(i).replace(OUTPUT_FILE_PATTERN, outFile.getAbsolutePath()));
            continue;
          } else {
            logger.error("{} pattern found, but no valid output filename was specified", OUTPUT_FILE_PATTERN);
            throw new ExecuteException(OUTPUT_FILE_PATTERN
                    + " pattern found, but no valid output filename was specified");
          }
        }
      }

      return runCommand(arguments, outFile, expectedType);

    } catch (IOException e) {
      logger.error("Error retrieving file from workspace: {}", element.getURI());
      throw new ExecuteException("Error retrieving file from workspace: " + element.getURI(), e);
    } catch (NotFoundException e) {
      logger.error("Element '{}' cannot be found in the workspace.", element.getURI());
      throw new ExecuteException("Element " + element.getURI() + " cannot be found in the workspace");
    }
  }

  private String runCommand(List<String> command, File outFile, Type expectedType) throws ExecuteException {

    Process p = null;
    int result = 0;

    try {
      logger.info("Running command {}", command.get(0));
      logger.debug("Starting subprocess {} with arguments {}", command.get(0), StringUtils.join(command.subList(1, command.size()), ", "));

      ProcessBuilder pb = new ProcessBuilder(command);
      pb.redirectErrorStream(true);

      p = pb.start();
      result = p.waitFor();

      logger.debug("Command {} finished with result {}", command.get(0), result);

      if (result == 0) {
        // Read the command output
        if (outFile != null) {
          if (outFile.isFile()) {
            URI newURI = workspace.putInCollection(COLLECTION, outFile.getName(), new FileInputStream(outFile));
            if (outFile.delete()) {
              logger.debug("Deleted the local copy of the encoded file at {}", outFile.getAbsolutePath());
            } else {
              logger.warn("Unable to delete the encoding output at {}", outFile.getAbsolutePath());
            }
            return MediaPackageElementParser.getAsXml(MediaPackageElementBuilderFactory.newInstance()
                    .newElementBuilder().elementFromURI(newURI, expectedType, null));
          }
        }
        return "";
      } else {
        // 'Scanner' reads tokens delimited by an specific character (set).
        // By telling a Scanner to use the 'beginning of the input boundary' character as delimiter, which of course
        // will never find, yields the whole String as the next token.
        String line;
        try {
          line = new Scanner(p.getInputStream()).useDelimiter("\\A").next();
        } catch (NoSuchElementException e) {
          line = "";
        }

        throw new ExecuteException(String.format("Process %s returned error code %d with this output:\n%s",
                command.get(0), result, line.trim()));
      }
    } catch (InterruptedException e) {
      throw new ExecuteException("The executor thread has been unexpectedly interrupted", e);
    } catch (IOException e) {
      // Only log the first argument, the executable, as other arguments may contain sensitive values
      // e.g. MySQL password/user, paths, etc. that should not be shown to caller
      logger.error("Could not start subprocess {}", command.get(0));
      throw new ExecuteException("Could not start subprocess: " + command.get(0), e);
    } catch (UnsupportedElementException e) {
      throw new ExecuteException("Couldn't create a new MediaPackage element of type " + expectedType.toString(), e);
    } catch (ConfigurationException e) {
      throw new ExecuteException("Couldn't instantiate a new MediaPackage element builder", e);
    } catch (MediaPackageException e) {
      throw new ExecuteException("Couldn't serialize a new Mediapackage element of type " + expectedType.toString(), e);
    } finally {
      IoSupport.closeQuietly(p);
    }
  }

  /**
   * Returns a list of strings broken on whitespace characters except where those whitespace characters are escaped or
   * quoted.
   *
   * @return list of individual arguments
   */
  private List<String> splitParameters(String input) {

    // This delimiter matches any non-escaped quote
    final String quoteDelim = "(?<!\\\\)\"";

    // This delimiter matches any number of non-escaped spaces
    final String spaceDelim = "((?<!\\\\)\\s)+";

    ArrayList<String> parsedInput = new ArrayList<String>();
    boolean quoted = false;

    for (String token1 : input.split(quoteDelim))
      if (quoted) {
        parsedInput.add(token1);
        quoted = false;
      } else {
        for (String token2 : token1.split(spaceDelim))
          // This ignores empty tokens if quotes are at the beginning or the end of the string
          if (!token2.isEmpty())
            parsedInput.add(token2);
        quoted = true;
      }

    return parsedInput;
  }

  /**
   * Sets the receipt service
   * 
   * @param serviceRegistry
   *          the service registry
   */
  public void setServiceRegistry(ServiceRegistry serviceRegistry) {
    this.serviceRegistry = serviceRegistry;
  }

  /**
   * {@inheritDoc}
   * 
   * @see org.opencastproject.job.api.AbstractJobProducer#getServiceRegistry()
   */
  @Override
  protected ServiceRegistry getServiceRegistry() {
    return serviceRegistry;
  }

  /**
   * {@inheritDoc}
   * 
   * @see org.opencastproject.job.api.AbstractJobProducer#getSecurityService()
   */
  @Override
  protected SecurityService getSecurityService() {
    return securityService;
  }

  /**
   * Callback for setting the security service.
   * 
   * @param securityService
   *          the securityService to set
   */
  public void setSecurityService(SecurityService securityService) {
    this.securityService = securityService;
  }

  /**
   * Callback for setting the user directory service.
   * 
   * @param userDirectoryService
   *          the userDirectoryService to set
   */
  public void setUserDirectoryService(UserDirectoryService userDirectoryService) {
    this.userDirectoryService = userDirectoryService;
  }

  /**
   * {@inheritDoc}
   * 
   * @see org.opencastproject.job.api.AbstractJobProducer#getUserDirectoryService()
   */
  @Override
  protected UserDirectoryService getUserDirectoryService() {
    return userDirectoryService;
  }

  /**
   * {@inheritDoc}
   * 
   * @see org.opencastproject.job.api.AbstractJobProducer#getOrganizationDirectoryService()
   */
  @Override
  protected OrganizationDirectoryService getOrganizationDirectoryService() {
    return organizationDirectoryService;
  }

  /**
   * Sets a reference to the organization directory service.
   * 
   * @param organizationDirectory
   *          the organization directory
   */
  public void setOrganizationDirectoryService(OrganizationDirectoryService organizationDirectory) {
    this.organizationDirectoryService = organizationDirectory;
  }

  /**
   * Sets a reference to the workspace service.
   * 
   * @param workspace
   */
  public void setWorkspace(Workspace workspace) {
    this.workspace = workspace;
  }

  @Override
  public void updated(@SuppressWarnings("rawtypes") Dictionary properties) throws org.osgi.service.cm.ConfigurationException {
    executeJobLoad = LoadUtil.getConfiguredLoadValue(properties, EXECUTE_JOB_LOAD_KEY, DEFAULT_EXECUTE_JOB_LOAD, serviceRegistry);
  }

}