TransferredResourcesScanner.java

/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE file at the root of the source
 * tree and available online at
 *
 * https://github.com/keeps/roda
 */
package org.roda.core.common.monitor;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.lang3.StringUtils;
import org.roda.core.RodaCoreFactory;
import org.roda.core.common.iterables.CloseableIterable;
import org.roda.core.data.common.RodaConstants;
import org.roda.core.data.exceptions.AlreadyExistsException;
import org.roda.core.data.exceptions.GenericException;
import org.roda.core.data.exceptions.IsStillUpdatingException;
import org.roda.core.data.exceptions.NotFoundException;
import org.roda.core.data.exceptions.RequestNotValidException;
import org.roda.core.data.v2.LiteRODAObject;
import org.roda.core.data.v2.common.OptionalWithCause;
import org.roda.core.data.v2.index.filter.Filter;
import org.roda.core.data.v2.index.filter.SimpleFilterParameter;
import org.roda.core.data.v2.ip.TransferredResource;
import org.roda.core.index.IndexService;
import org.roda.core.model.LiteRODAObjectFactory;
import org.roda.core.storage.ContentPayload;
import org.roda.core.storage.fs.FSUtils;
import org.roda.core.util.IdUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TransferredResourcesScanner {
  private static final Logger LOGGER = LoggerFactory.getLogger(TransferredResourcesScanner.class);
  private static final List<String> fieldsToReturn = Arrays.asList(RodaConstants.INDEX_UUID,
    RodaConstants.TRANSFERRED_RESOURCE_RELATIVEPATH);

  private final Path basePath;
  private IndexService index;

  public TransferredResourcesScanner(Path basePath, IndexService index) {
    this.basePath = basePath;
    this.index = index;
  }

  public void commit() throws GenericException {
    index.commit(TransferredResource.class);
  }

  public Path getBasePath() {
    return basePath;
  }

  public TransferredResource createFolder(String parentUUID, String folderName)
    throws GenericException, RequestNotValidException, NotFoundException {
    Path parentPath;
    if (parentUUID != null) {
      TransferredResource parent = index.retrieve(TransferredResource.class, parentUUID, fieldsToReturn);
      parentPath = basePath.resolve(parent.getRelativePath());
    } else {
      parentPath = basePath;
    }

    try {
      Path createdPath = Files.createDirectories(parentPath.resolve(folderName));
      BasicFileAttributes attrs = Files.readAttributes(createdPath, BasicFileAttributes.class);
      TransferredResource resource = createTransferredResource(createdPath, attrs, 0L, basePath, new Date());
      index.create(TransferredResource.class, resource);
      return resource;
    } catch (IOException e) {
      LOGGER.error("Cannot create folder", e);
      throw new GenericException("Cannot create folder", e);
    }
  }

  public TransferredResource createFile(String parentUUID, String fileName, InputStream inputStream)
    throws GenericException, RequestNotValidException, NotFoundException, AlreadyExistsException {
    Path parentPath;
    if (StringUtils.isNotBlank(parentUUID)) {
      TransferredResource parent = index.retrieve(TransferredResource.class, parentUUID, fieldsToReturn);
      parentPath = basePath.resolve(parent.getRelativePath());
    } else {
      parentPath = basePath;
    }

    Path file = parentPath.resolve(fileName);
    try {
      try {
        Files.createDirectories(parentPath);
      } catch (FileAlreadyExistsException e) {
        // do nothing and carry on
      }

      Files.copy(inputStream, file);
      BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class);
      TransferredResource resource = createTransferredResource(file, attrs, attrs.size(), basePath, new Date());
      index.create(TransferredResource.class, resource);
      return resource;
    } catch (FileAlreadyExistsException e) {
      LOGGER.error("Cannot create file", e);
      throw new AlreadyExistsException(file.toString());
    } catch (IOException e) {
      LOGGER.error("Cannot create file", e);
      throw new GenericException("Cannot create file", e);
    }
  }

  public InputStream retrieveFile(String path) throws NotFoundException, RequestNotValidException, GenericException {
    InputStream ret;
    Path p = basePath.resolve(path);
    if (!FSUtils.exists(p)) {
      throw new NotFoundException("File not found: " + path);
    } else if (!FSUtils.isFile(p)) {
      throw new RequestNotValidException("Requested file is not a regular file: " + path);
    } else {
      try {
        ret = Files.newInputStream(p);
      } catch (IOException e) {
        throw new GenericException("Could not create input stream: " + e.getMessage());
      }
    }
    return ret;
  }

  protected static TransferredResource createTransferredResource(Path resourcePath, BasicFileAttributes attr, long size,
    Path basePath, Date lastScanDate) {
    Date d = new Date(attr.creationTime().toMillis());

    TransferredResource tr = instantiateTransferredResource(resourcePath, basePath);
    tr.setSize(size);
    tr.setCreationDate(d);
    tr.setLastScanDate(lastScanDate);

    return tr;
  }

  public static TransferredResource instantiateTransferredResource(Path resourcePath, Path basePath) {
    Path relativeToBase = basePath.relativize(resourcePath);
    TransferredResource tr = new TransferredResource();

    tr.setFile(!FSUtils.isDirectory(resourcePath));
    tr.setFullPath(resourcePath.toString());
    String id = relativeToBase.toString();
    tr.setId(id);
    tr.setUUID(IdUtils.getTransferredResourceUUID(relativeToBase));
    tr.setName(resourcePath.getFileName().toString());

    tr.setRelativePath(relativeToBase.toString());
    if (relativeToBase.getParent() != null) {
      String parentId = relativeToBase.getParent().toString();
      tr.setParentId(parentId);
      tr.setParentUUID(IdUtils.createUUID(parentId));
    }

    List<String> ancestors = new ArrayList<>();

    StringBuilder temp = new StringBuilder();
    Iterator<Path> pathIterator = relativeToBase.iterator();
    while (pathIterator.hasNext()) {
      temp.append(pathIterator.next().toString());
      ancestors.add(temp.toString());
      temp.append("/");
    }
    ancestors.remove(ancestors.size() - 1);
    tr.setAncestorsPaths(ancestors);

    return tr;
  }

  public void deleteTransferredResource(List<String> ids)
    throws NotFoundException, GenericException, RequestNotValidException {
    for (String uuid : ids) {
      TransferredResource tr = index.retrieve(TransferredResource.class, uuid, fieldsToReturn);
      Path relative = Paths.get(tr.getRelativePath());
      Path fullPath = basePath.resolve(relative);
      if (FSUtils.exists(fullPath)) {
        FSUtils.deletePath(fullPath);

        Filter filter = new Filter(
          new SimpleFilterParameter(RodaConstants.TRANSFERRED_RESOURCE_ANCESTORS, relative.toString()));
        index.delete(TransferredResource.class, filter);
      } else {
        throw new NotFoundException("Path does not exist: " + fullPath);
      }
    }
    index.delete(TransferredResource.class, ids);
    index.commit(TransferredResource.class);
  }

  public void updateTransferredResources(Optional<String> folderRelativePath, boolean waitToFinish)
    throws IsStillUpdatingException, GenericException {
    if (!RodaCoreFactory.getTransferredResourcesScannerUpdateStatus(folderRelativePath)) {
      if (index != null) {
        ReindexTransferredResourcesRunnable reindexRunnable = new ReindexTransferredResourcesRunnable(index, basePath,
          folderRelativePath);

        if (waitToFinish) {
          reindexRunnable.run();
        } else {
          Thread threadReindex = new Thread(reindexRunnable, "ReindexThread");
          threadReindex.start();
        }
      } else {
        throw new GenericException("Could not update transferred resources because index was not initialized");
      }
    } else {
      LOGGER.warn("Could not update transferred resources because it is still updating");
      throw new IsStillUpdatingException();
    }
  }

  public void updateTransferredResource(Optional<String> folderRelativePath, ContentPayload payload, String name,
    boolean waitToFinish) throws NotFoundException, GenericException, IOException, IsStillUpdatingException {
    if (folderRelativePath.isPresent()) {
      Path path = basePath.resolve(folderRelativePath.get());
      Path parent = path.getParent();
      Path parentToBase = basePath.relativize(parent);
      FSUtils.deletePath(path);

      payload.writeToPath(parent.resolve(name));
      updateTransferredResources(Optional.ofNullable(parentToBase.toString()), waitToFinish);
    }
  }

  public String renameTransferredResource(TransferredResource resource, String newName, boolean replaceExisting,
    boolean reindexResources)
    throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {

    if (FSUtils.exists(Paths.get(resource.getFullPath()))) {
      Path resourcePath = Paths.get(resource.getFullPath());
      Path newPath = resourcePath.getParent().resolve(newName);
      FSUtils.move(resourcePath, newPath, replaceExisting);

      if (reindexResources) {
        if (resource.getParentUUID() != null) {
          try {
            TransferredResource parent = index.retrieve(TransferredResource.class, resource.getParentUUID(),
              fieldsToReturn);
            if (parent != null) {
              updateTransferredResources(Optional.of(parent.getRelativePath()), true);
            } else {
              updateTransferredResources(Optional.empty(), true);
            }
          } catch (GenericException | NotFoundException e) {
            LOGGER.error("Could not reindex transferred resources after renaming");
          }
        } else {
          updateTransferredResources(Optional.empty(), true);
        }
      }

      Path relativeToBase = basePath.relativize(resourcePath.getParent().resolve(newName));
      return IdUtils.createUUID(relativeToBase.toString());
    } else {
      throw new NotFoundException("Transferred resource was moved or does not exist");
    }
  }

  public Map<String, String> moveTransferredResource(String newRelativePath, List<String> resourcesUUIDs,
    boolean replaceExisting)
    throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {
    List<TransferredResource> resources = Collections.emptyList();
    try {
      List<String> resourceFields = Arrays.asList(RodaConstants.INDEX_UUID, RodaConstants.TRANSFERRED_RESOURCE_FULLPATH,
        RodaConstants.TRANSFERRED_RESOURCE_RELATIVEPATH, RodaConstants.TRANSFERRED_RESOURCE_NAME);
      resources = index.retrieve(TransferredResource.class, resourcesUUIDs, resourceFields);
    } catch (NotFoundException e) {
      // do nothing and pass it an empty list
    }
    return moveTransferredResource(resources, newRelativePath, replaceExisting, false, true);
  }

  public Map<String, String> moveTransferredResource(List<TransferredResource> resources, String newRelativePath,
    boolean replaceExisting, boolean reindexResources)
    throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {
    return moveTransferredResource(resources, newRelativePath, replaceExisting, reindexResources, false);
  }

  public Map<String, String> moveTransferredResource(List<TransferredResource> resources, String newRelativePath,
    boolean replaceExisting, boolean reindexResources, boolean addOldRelativePathToNewRelativePath)
    throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {

    Map<String, String> oldToNewTransferredResourceIds = new HashMap<>();
    List<TransferredResource> resourcesToIndex = new ArrayList<>();
    boolean notFoundResources = false;

    String baseFolder = RodaCoreFactory.getRodaConfiguration().getString("core.ingest.processed.base_folder",
      "PROCESSED");
    String successFolder = RodaCoreFactory.getRodaConfiguration()
      .getString("core.ingest.processed.successfully_ingested", "SUCCESSFULLY_INGESTED");
    String unsuccessFolder = RodaCoreFactory.getRodaConfiguration()
      .getString("core.ingest.processed.unsuccessfully_ingested", "UNSUCCESSFULLY_INGESTED");

    for (TransferredResource resource : resources) {
      if (FSUtils.exists(Paths.get(resource.getFullPath()))) {
        Path newResourcePath = basePath.resolve(newRelativePath);
        if (addOldRelativePathToNewRelativePath) {
          newResourcePath = newResourcePath.resolve(resource.getRelativePath()
            .replace(baseFolder + "/" + successFolder + "/", "").replace(baseFolder + "/" + unsuccessFolder + "/", ""));
        } else {
          newResourcePath = newResourcePath.resolve(resource.getName());
        }

        FSUtils.move(Paths.get(resource.getFullPath()), newResourcePath, replaceExisting);

        // create & index transferred resource in the new location
        TransferredResource newResource = instantiateTransferredResource(newResourcePath, basePath);
        Date creationDate = resource.getCreationDate();
        try {
          BasicFileAttributes attr = Files.readAttributes(newResourcePath, BasicFileAttributes.class);
          creationDate = new Date(attr.creationTime().toMillis());
        } catch (IOException e) {
          creationDate = new Date();
        }

        newResource.setCreationDate(creationDate);
        newResource.setSize(resource.getSize());
        newResource.setLastScanDate(new Date());
        try {
          index.create(TransferredResource.class, newResource);
        } catch (RequestNotValidException e) {
          // do nothing
        }

        oldToNewTransferredResourceIds.put(resource.getUUID(), newResource.getUUID());
        resourcesToIndex.add(resource);
      } else {
        notFoundResources = true;
      }
    }

    if (reindexResources) {
      updateTransferredResources(Optional.of(newRelativePath), true);
    }
    reindexOldResourcesParentsAfterMove(resourcesToIndex);

    // doing the throw after the moving process to reindex the moved ones
    if (notFoundResources) {
      throw new NotFoundException("Some transferred resources were moved or do not exist");
    }

    return oldToNewTransferredResourceIds;
  }

  public void reindexOldResourcesParentsAfterMove(List<TransferredResource> resources)
    throws IsStillUpdatingException, GenericException {

    try {
      List<String> resourceUUIDs = resources.stream().map(tr -> tr.getUUID()).collect(Collectors.toList());
      index.delete(TransferredResource.class, resourceUUIDs);
    } catch (RequestNotValidException e) {
      LOGGER.error("Could not delete old transferred resources");
    }
  }

  public CloseableIterable<OptionalWithCause<LiteRODAObject>> listTransferredResources() {
    CloseableIterable<OptionalWithCause<LiteRODAObject>> resources = null;

    try {
      final Stream<Path> files = Files.walk(basePath, FileVisitOption.FOLLOW_LINKS)
        .filter(path -> !path.equals(basePath));
      final Iterator<Path> fileIterator = files.iterator();

      resources = new CloseableIterable<OptionalWithCause<LiteRODAObject>>() {
        @Override
        public void close() throws IOException {
          files.close();
        }

        @Override
        public Iterator<OptionalWithCause<LiteRODAObject>> iterator() {

          return new Iterator<OptionalWithCause<LiteRODAObject>>() {
            @Override
            public boolean hasNext() {
              return fileIterator.hasNext();
            }

            @Override
            public OptionalWithCause<LiteRODAObject> next() {
              Path file = fileIterator.next();
              Optional<LiteRODAObject> liteResource = LiteRODAObjectFactory.get(TransferredResource.class,
                Arrays.asList(file.toString()), false);
              return OptionalWithCause.of(liteResource);
            }
          };
        }
      };
    } catch (IOException e) {
      LOGGER.error("Errored when file walking to list transferred resources");
    }

    return resources;
  }

}