OaiPmhPublicationServiceImpl.java

/**
 * Licensed to The Apereo Foundation under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 *
 * The Apereo Foundation licenses this file to you under the Educational
 * Community License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License
 * at:
 *
 *   http://opensource.org/licenses/ecl2.txt
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 */
package org.opencastproject.publication.oaipmh;

import static com.entwinemedia.fn.Stream.$;
import static java.lang.String.format;
import static org.opencastproject.mediapackage.MediaPackageSupport.Filters.ofChannel;
import static org.opencastproject.util.JobUtil.waitForJobs;
import static org.opencastproject.util.data.Collections.set;
import static org.opencastproject.util.data.Monadics.mlist;

import org.opencastproject.distribution.api.DistributionException;
import org.opencastproject.distribution.api.DistributionService;
import org.opencastproject.distribution.api.DownloadDistributionService;
import org.opencastproject.job.api.AbstractJobProducer;
import org.opencastproject.job.api.Job;
import org.opencastproject.mediapackage.MediaPackage;
import org.opencastproject.mediapackage.MediaPackageElement;
import org.opencastproject.mediapackage.MediaPackageElementParser;
import org.opencastproject.mediapackage.MediaPackageException;
import org.opencastproject.mediapackage.MediaPackageParser;
import org.opencastproject.mediapackage.MediaPackageReference;
import org.opencastproject.mediapackage.MediaPackageReferenceImpl;
import org.opencastproject.mediapackage.MediaPackageSupport;
import org.opencastproject.mediapackage.Publication;
import org.opencastproject.mediapackage.PublicationImpl;
import org.opencastproject.oaipmh.persistence.OaiPmhDatabase;
import org.opencastproject.oaipmh.persistence.OaiPmhDatabaseException;
import org.opencastproject.oaipmh.persistence.QueryBuilder;
import org.opencastproject.oaipmh.persistence.SearchResult;
import org.opencastproject.oaipmh.persistence.SearchResultItem;
import org.opencastproject.oaipmh.server.OaiPmhServerInfo;
import org.opencastproject.oaipmh.server.OaiPmhServerInfoUtil;
import org.opencastproject.publication.api.OaiPmhPublicationService;
import org.opencastproject.publication.api.PublicationException;
import org.opencastproject.security.api.OrganizationDirectoryService;
import org.opencastproject.security.api.SecurityService;
import org.opencastproject.security.api.UserDirectoryService;
import org.opencastproject.serviceregistry.api.ServiceRegistry;
import org.opencastproject.serviceregistry.api.ServiceRegistryException;
import org.opencastproject.util.MimeTypes;
import org.opencastproject.util.NotFoundException;
import org.opencastproject.util.UrlSupport;
import org.opencastproject.util.data.Function;
import org.opencastproject.util.data.Option;
import org.opencastproject.util.data.functions.Functions;

import com.entwinemedia.fn.P2;
import com.entwinemedia.fn.Products;

import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.http.client.utils.URIUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;

/**
 * Publishes a recording to an OAI-PMH publication repository.
 */
public class OaiPmhPublicationServiceImpl extends AbstractJobProducer implements OaiPmhPublicationService {
  /** Logging facility */
  private static final Logger logger = LoggerFactory.getLogger(OaiPmhPublicationServiceImpl.class);

  /** The publication element mime type */
  private static final String MIME_TYPE = "text/xml";

  /** The element id separator */
  private static final String SEPARATOR = ";;";

  /** List of available operations on jobs */
  private enum Operation {
    Publish, Retract
  }

  /** The remote service registry */
  private ServiceRegistry serviceRegistry = null;

  /** The security service */
  private SecurityService securityService = null;

  /** The user directory service */
  private UserDirectoryService userDirectoryService = null;

  /** The organization directory service */
  private OrganizationDirectoryService organizationDirectoryService = null;

  /** The download distribution service */
  private DownloadDistributionService downloadDistributionService = null;

  /** The streaming distribution service */
  private DistributionService streamingDistributionService = null;

  /** The OAI-PMH persistence */
  private OaiPmhDatabase persistence = null;

  private OaiPmhServerInfo oaiPmhServerInfo;

  /**
   * Callback for the OSGi environment to set the service registry reference.
   *
   * @param serviceRegistry
   *          the service registry
   */
  protected void setServiceRegistry(ServiceRegistry serviceRegistry) {
    this.serviceRegistry = serviceRegistry;
  }

  /**
   * Callback for setting the security service.
   *
   * @param securityService
   *          the securityService to set
   */
  public void setSecurityService(SecurityService securityService) {
    this.securityService = securityService;
  }

  /**
   * Callback for setting the user directory service.
   *
   * @param userDirectoryService
   *          the userDirectoryService to set
   */
  public void setUserDirectoryService(UserDirectoryService userDirectoryService) {
    this.userDirectoryService = userDirectoryService;
  }

  /**
   * Sets a reference to the organization directory service.
   *
   * @param organizationDirectory
   *          the organization directory
   */
  public void setOrganizationDirectoryService(OrganizationDirectoryService organizationDirectory) {
    this.organizationDirectoryService = organizationDirectory;
  }

  /**
   * Callback for the OSGi declarative services configuration.
   *
   * @param distributionService
   *          the distribution service
   */
  public void setDownloadDistributionService(DownloadDistributionService distributionService) {
    this.downloadDistributionService = distributionService;
  }

  /**
   * Callback for the OSGi declarative services configuration.
   *
   * @param distributionService
   *          the distribution service
   */
  public void setStreamingDistributionService(DistributionService distributionService) {
    this.streamingDistributionService = distributionService;
  }

  /**
   * Callback for the OSGi declarative services configuration.
   *
   * @param persistence
   *          the OAI-PMH persistence
   */
  public void setPersistence(OaiPmhDatabase persistence) {
    this.persistence = persistence;
  }

  /** OSGi DI. */
  public void setOaiPmhServerInfo(OaiPmhServerInfo oaiPmhServerInfo) {
    this.oaiPmhServerInfo = oaiPmhServerInfo;
  }

  /**
   * Creates a new instance of the OAI-PMH publication service.
   */
  public OaiPmhPublicationServiceImpl() {
    super(JOB_TYPE);
  }

  @Override
  public Job publish(MediaPackage mediaPackage, String repository, Set<String> downloadIds, Set<String> streamingIds,
          boolean checkAvailability) throws PublicationException, MediaPackageException {
    if (mediaPackage == null)
      throw new MediaPackageException("Media package must be specified");
    if (StringUtils.isEmpty(repository))
      throw new IllegalStateException("Repository must be specified");

    try {
      return serviceRegistry.createJob(JOB_TYPE, Operation.Publish.toString(),
              Arrays.asList(MediaPackageParser.getAsXml(mediaPackage), // 0
                      repository, // 1
                      StringUtils.join(downloadIds, SEPARATOR), // 2
                      StringUtils.join(streamingIds, SEPARATOR), // 3
                      Boolean.toString(checkAvailability))); // 4
    } catch (ServiceRegistryException e) {
      throw new PublicationException("Unable to create a job", e);
    }
  }

  protected Publication publishInternal(Job job, MediaPackage mp, String repository, Set<String> downloadIds,
          Set<String> streamingIds, boolean checkAvailability) throws PublicationException, MediaPackageException {
    if (!oaiPmhServerInfo.hasRepo(repository)) {
      final String msg = format("OAI-PMH repository %s does not exist", repository);
      logger.error(msg);
      throw new PublicationException(msg);
    }
    try {
      logger.info("Publishing media package {} to OAI-PMH", mp.getIdentifier());
      // => HIWEST-1692 avoid duplication of distribution artifacts
      // - Retract an existing publication.
      // - Check if media package has been published before before running a retract.
      //   Otherwise error messages will be logged.
      if (hasBeenPublished(mp.getIdentifier().toString(), repository)) {
        retractInternal(job, mp, repository);
      }
      final Publication publication = createPublicationElement(mp.getIdentifier().compact(), repository);
      final MediaPackage mpPublication = publishElementsToDownload(job, mp, repository, downloadIds, streamingIds,
              checkAvailability);
      if (mpPublication == null) {
        return null;
      }
      mpPublication.add(publication);
      try {
        persistence.store(mpPublication, repository);
        logger.info("Published {} to OAI-PMH repository {}", mpPublication, repository);
      } catch (OaiPmhDatabaseException e) {
        logger.error("Unable to store '{}' to OAI-PMH repository '{}'", mp, repository);
        throw new PublicationException(e);
      }
      logger.debug("Publish operation complete");
      return publication;

    } catch (PublicationException e) {
      throw e;
    } catch (Exception e) {
      throw new PublicationException(e);
    }
  }

  /**
   * Check if media package {@code mpId} has already been published to {@code repository}.
   */
  private boolean hasBeenPublished(String mpId, String repository) {
    return persistence.search(
        QueryBuilder
            .queryRepo(repository)
            .mediaPackageId(mpId)
            .build())
        .size() > 0;
  }

  /** Create a new publication element. */
  private Publication createPublicationElement(String mpId, String repository) throws PublicationException {
    for (String hostUrl : OaiPmhServerInfoUtil.oaiPmhServerUrlOfCurrentOrganization(securityService)) {
      final URI engageUri = URIUtils.resolve(
              URI.create(UrlSupport.concat(hostUrl, oaiPmhServerInfo.getMountPoint(), repository)),
              "?verb=ListMetadataFormats&identifier=" + mpId);
      return PublicationImpl.publication(UUID.randomUUID().toString(), publicationChannelId(repository), engageUri,
              MimeTypes.parseMimeType(MIME_TYPE));
    }
    // no host URL
    final String msg = format("No host url for oai-pmh server configured for organization %s " + "("
            + OaiPmhServerInfoUtil.ORG_CFG_OAIPMH_SERVER_HOSTURL + ")", securityService.getOrganization().getId());
    logger.error(msg);
    throw new PublicationException(msg);
  }

  @Override
  public Job retract(MediaPackage mediaPackage, String repository) throws PublicationException {
    if (mediaPackage == null)
      throw new IllegalArgumentException("Media package must be specified");
    try {
      return serviceRegistry.createJob(JOB_TYPE, Operation.Retract.toString(),
              Arrays.asList(MediaPackageParser.getAsXml(mediaPackage), // 0
                      repository)); // 1
    } catch (ServiceRegistryException e) {
      throw new PublicationException("Unable to create a job", e);
    }
  }

  protected MediaPackage publishElementsToDownload(Job parentJob, MediaPackage mediaPackage, String repository,
          Set<String> downloadIds, Set<String> streamingIds, boolean checkAvailability)
          throws PublicationException, MediaPackageException {
    // Distribute to download
    final List<P2<Job, String>> jobs = new ArrayList<>();
    final String pubChannelId = publicationChannelId(repository);
    try {
      for (String elementId : downloadIds) {
        Job job = downloadDistributionService.distribute(pubChannelId, mediaPackage, elementId, checkAvailability);
        if (job == null)
          continue;
        jobs.add(Products.E.p2(job, elementId));
      }
      for (String elementId : streamingIds) {
        Job job = streamingDistributionService.distribute(pubChannelId, mediaPackage, elementId);
        if (job == null)
          continue;
        jobs.add(Products.E.p2(job, elementId));
      }
    } catch (DistributionException e) {
      throw new PublicationException(e);
    }

    if (jobs.size() < 1) {
      logger.info("No media package element was found for distribution to engage");
      return null;
    }

    // Wait until all distribution jobs have returned
    final List<Job> waitForJobs = $(jobs).map(com.entwinemedia.fn.fns.Products.<Job>p2_1()).toList();
    if (!waitForJobs(parentJob, serviceRegistry, waitForJobs).isSuccess())
      throw new PublicationException("One of the distribution jobs did not complete successfully");

    logger.debug("Distribute operation completed");

    try {
      return getMediaPackageForOaiPmh(mediaPackage, jobs);
    } catch (Throwable t) {
      throw new PublicationException(t);
    }
  }

  /**
   * Retracts the mediapackage with the given identifier from the OAI-PMH repository.
   *
   * @return the retracted element or <code>null</code> if the element was not retracted
   */
  protected Publication retractInternal(Job job, MediaPackage mediapackage, String repository)
          throws PublicationException {
    final String mediapackageId = mediapackage.getIdentifier().toString();
    logger.info("Trying to retract media package '{}' from OAI-PMH repository '{}'", mediapackageId, repository);
    // get _publicized_ media package
    final SearchResult sr = persistence.search(QueryBuilder.queryRepo(repository).mediaPackageId(mediapackageId)
            .build());
    final String pubChannelId = publicationChannelId(repository);
    // iterate the result set (which has at most 1 element)
    for (SearchResultItem item : sr.getItems()) {
      final MediaPackage mp = item.getMediaPackage();
      // remove from database
      try {
        persistence.delete(mediapackageId, repository);
      } catch (OaiPmhDatabaseException e) {
        logger.error("Unable to delete mediapackage '{}' from OAI-PMH repository '{}'", mediapackageId, repository);
        throw new PublicationException(e);
      } catch (NotFoundException e) {
        logger.warn("Unable to remove mediapackage '{}'from OAI-PMH repository '{}' since it does not exist",
                mediapackageId, repository);
      }
      // retract all media package elements from their distribution location
      final List<Job> jobs = mlist(mp.getElements()).filter(MediaPackageSupport.Filters.isNotPublication)
              .bind(retractFromDistribution(pubChannelId, mp)).value();
      if (!waitForJobs(job, serviceRegistry, jobs).isSuccess())
        throw new PublicationException(format("Unable to retract an element of mediapackage '%s' from the "
                + "distribution for publication repository '%s'", mediapackageId, pubChannelId));
      // use the media package parameter to determine the publication element since
      // the stored one does not contain it.
      // todo: design flaw
      for (Publication p : mlist(mediapackage.getPublications()).find(ofChannel(pubChannelId)))
        return p;
      logger.warn(format("Database query for mediapackage %s, publication repository %s yielded a mediapackage"
              + " but it does not contain a matching publication element", mediapackageId, pubChannelId));
    }
    return null;
  }

  @SuppressWarnings("unchecked")
  private Function<MediaPackageElement, List<Job>> retractFromDistribution(final String channelId, final MediaPackage mp) {
    return new Function.X<MediaPackageElement, List<Job>>() {
      @Override
      public List<Job> xapply(MediaPackageElement mpe) throws Exception {
        return mlist(Option.some(downloadDistributionService.retract(channelId, mp, mpe.getIdentifier())),
                Option.option(streamingDistributionService.retract(channelId, mp, mpe.getIdentifier()))).flatMap(
                Functions.<Option<Job>> identity()).value();
      }
    };
  }

  private static String publicationChannelId(String repository) {
    return PUBLICATION_CHANNEL_PREFIX.concat(repository);
  }

  @Override
  protected String process(Job job) throws Exception {
    Operation op = null;
    String operation = job.getOperation();
    List<String> arguments = job.getArguments();
    try {
      op = Operation.valueOf(operation);
      MediaPackage mediaPackage = MediaPackageParser.getFromXml(arguments.get(0));
      String repository = arguments.get(1);
      switch (op) {
        case Publish:
          final Set<String> downloadIds = set(StringUtils.split(arguments.get(2), SEPARATOR));
          final Set<String> streamingIds = set(StringUtils.split(arguments.get(3), SEPARATOR));
          boolean checkAvailability = BooleanUtils.toBoolean(arguments.get(4));
          MediaPackageElement publishedElement = publishInternal(job, mediaPackage, repository, downloadIds,
                  streamingIds, checkAvailability);
          return (publishedElement != null) ? MediaPackageElementParser.getAsXml(publishedElement) : null;
        case Retract:
          MediaPackageElement retractedElement = retractInternal(job, mediaPackage, repository);
          return (retractedElement != null) ? MediaPackageElementParser.getAsXml(retractedElement) : null;
        default:
          throw new IllegalStateException("Don't know how to handle operation '" + operation + "'");
      }
    } catch (IllegalArgumentException e) {
      throw new ServiceRegistryException("This service can't handle operations of type '" + op + "'", e);
    } catch (IndexOutOfBoundsException e) {
      throw new ServiceRegistryException("This argument list for operation '" + op + "' does not meet expectations", e);
    } catch (Exception e) {
      logger.error("Error processing OAI-PMH operation job {}: {}", job.getId(), ExceptionUtils.getStackTrace(e));
      throw new ServiceRegistryException("Error handling operation '" + op + "'", e);
    }
  }

  /**
   * Returns a media package that only contains elements that are marked for distribution.
   *
   * @param current
   *          the current mediapackage
   * @param jobs
   *          list of distribution jobs and their distributed element id
   * @return the new mediapackage
   */
  protected MediaPackage getMediaPackageForOaiPmh(MediaPackage current, List<P2<Job, String>> jobs) throws MediaPackageException,
          NotFoundException, ServiceRegistryException {
    MediaPackage mp = (MediaPackage) current.clone();

    // All the jobs have passed, let's update the mediapackage with references to the distributed elements
    List<String> elementsToPublish = new ArrayList<>();
    Map<String, String> distributedElementIds = new HashMap<>();

    for (P2<Job, String> entry : jobs) {
      Job job = serviceRegistry.getJob(entry.get1().getId());
      String sourceElementId = entry.get2();
      MediaPackageElement sourceElement = mp.getElementById(sourceElementId);

      // If there is no payload, then the item has not been distributed.
      if (job.getPayload() == null)
        continue;

      final MediaPackageElement distributedElement = MediaPackageElementParser.getFromXml(job.getPayload());

      // If the job finished successfully, but returned no new element, the channel simply doesn't support this
      // kind of element. So we just keep on looping.
      if (distributedElement == null)
        continue;

      // Make sure the mediapackage is prompted to create a new identifier for this element
      distributedElement.setIdentifier(null);

      // Copy references from the source elements to the distributed elements
      MediaPackageReference ref = sourceElement.getReference();
      if (ref != null && mp.getElementByReference(ref) != null) {
        MediaPackageReference newReference = (MediaPackageReference) ref.clone();
        distributedElement.setReference(newReference);
      }

      // Add the new element to the mediapackage
      mp.add(distributedElement);
      elementsToPublish.add(distributedElement.getIdentifier());
      distributedElementIds.put(sourceElementId, distributedElement.getIdentifier());

    }

    // Mark everything that is set for removal
    List<MediaPackageElement> removals = new ArrayList<>();
    for (MediaPackageElement element : mp.getElements()) {
      if (!elementsToPublish.contains(element.getIdentifier())) {
        removals.add(element);
      }
    }

    // Translate references to the distributed artifacts
    for (MediaPackageElement element : mp.getElements()) {

      if (removals.contains(element))
        continue;

      // Is the element referencing anything?
      MediaPackageReference reference = element.getReference();
      if (reference == null)
        continue;

      // See if the element has been distributed
      String distributedElementId = distributedElementIds.get(reference.getIdentifier());
      if (distributedElementId == null)
        continue;

      MediaPackageReference translatedReference = new MediaPackageReferenceImpl(mp.getElementById(distributedElementId));
      if (reference.getProperties() != null) {
        translatedReference.getProperties().putAll(reference.getProperties());
      }

      // Set the new reference
      element.setReference(translatedReference);

    }

    // Remove everything we don't want to add to publish except the publications
    for (MediaPackageElement element : removals) {
      if (MediaPackageElement.Type.Publication.equals(element.getElementType()))
        continue;
      mp.remove(element);
    }
    return mp;
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.job.api.AbstractJobProducer#getServiceRegistry()
   */
  @Override
  protected ServiceRegistry getServiceRegistry() {
    return serviceRegistry;
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.job.api.AbstractJobProducer#getSecurityService()
   */
  @Override
  protected SecurityService getSecurityService() {
    return securityService;
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.job.api.AbstractJobProducer#getUserDirectoryService()
   */
  @Override
  protected UserDirectoryService getUserDirectoryService() {
    return userDirectoryService;
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.job.api.AbstractJobProducer#getOrganizationDirectoryService()
   */
  @Override
  protected OrganizationDirectoryService getOrganizationDirectoryService() {
    return organizationDirectoryService;
  }

}