RodaCoreFactory.java

/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE file at the root of the source
 * tree and available online at
 *
 * https://github.com/keeps/roda
 */
package org.roda.core;

import java.io.BufferedReader;
import java.io.Console;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import javax.xml.validation.Schema;

import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.reflections.Reflections;
import org.reflections.scanners.ResourcesScanner;
import org.reflections.util.ClasspathHelper;
import org.reflections.util.ConfigurationBuilder;
import org.reflections.util.FilterBuilder;
import org.roda.core.common.LdapUtility;
import org.roda.core.common.Messages;
import org.roda.core.common.RodaUtils;
import org.roda.core.common.UserUtility;
import org.roda.core.common.iterables.CloseableIterable;
import org.roda.core.common.monitor.TransferUpdateStatus;
import org.roda.core.common.monitor.TransferredResourcesScanner;
import org.roda.core.data.common.RodaConstants;
import org.roda.core.data.common.RodaConstants.NodeType;
import org.roda.core.data.common.RodaConstants.PreservationAgentType;
import org.roda.core.data.common.RodaConstants.PreservationEventType;
import org.roda.core.data.common.RodaConstants.SolrType;
import org.roda.core.data.common.RodaConstants.StorageType;
import org.roda.core.data.exceptions.AlreadyExistsException;
import org.roda.core.data.exceptions.AuthorizationDeniedException;
import org.roda.core.data.exceptions.GenericException;
import org.roda.core.data.exceptions.IllegalOperationException;
import org.roda.core.data.exceptions.NotFoundException;
import org.roda.core.data.exceptions.RequestNotValidException;
import org.roda.core.data.exceptions.RoleAlreadyExistsException;
import org.roda.core.data.v2.common.Pair;
import org.roda.core.data.v2.index.IndexResult;
import org.roda.core.data.v2.index.facet.Facets;
import org.roda.core.data.v2.index.filter.BasicSearchFilterParameter;
import org.roda.core.data.v2.index.filter.EmptyKeyFilterParameter;
import org.roda.core.data.v2.index.filter.Filter;
import org.roda.core.data.v2.index.filter.FilterParameter;
import org.roda.core.data.v2.index.filter.SimpleFilterParameter;
import org.roda.core.data.v2.index.sort.Sorter;
import org.roda.core.data.v2.index.sublist.Sublist;
import org.roda.core.data.v2.ip.IndexedFile;
import org.roda.core.data.v2.ip.TransferredResource;
import org.roda.core.data.v2.ip.metadata.IndexedPreservationAgent;
import org.roda.core.data.v2.ip.metadata.IndexedPreservationEvent;
import org.roda.core.data.v2.user.Group;
import org.roda.core.data.v2.user.RODAMember;
import org.roda.core.data.v2.user.User;
import org.roda.core.index.IndexService;
import org.roda.core.index.utils.SolrUtils;
import org.roda.core.migration.MigrationManager;
import org.roda.core.model.ModelService;
import org.roda.core.plugins.PluginManager;
import org.roda.core.plugins.PluginManagerException;
import org.roda.core.plugins.PluginOrchestrator;
import org.roda.core.plugins.orchestrate.AkkaDistributedPluginOrchestrator;
import org.roda.core.plugins.orchestrate.AkkaEmbeddedPluginOrchestrator;
import org.roda.core.plugins.orchestrate.akka.distributed.AkkaDistributedPluginWorker;
import org.roda.core.storage.DefaultStoragePath;
import org.roda.core.storage.Resource;
import org.roda.core.storage.StorageService;
import org.roda.core.storage.fedora.FedoraStorageService;
import org.roda.core.storage.fs.FSUtils;
import org.roda.core.storage.fs.FileStorageService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.MetricRegistry;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;

import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;

/**
 * @author Hélder Silva <hsilva@keep.pt>
 */
public class RodaCoreFactory {

  private static final Logger LOGGER = LoggerFactory.getLogger(RodaCoreFactory.class);

  private static boolean instantiated = false;
  private static boolean instantiatedWithoutErrors = true;
  private static NodeType nodeType;
  private static boolean migrationMode = false;
  private static List<Path> toDeleteDuringShutdown = new ArrayList<>();

  // Core related objects
  private static Path rodaHomePath;
  private static Path storagePath;
  private static Path indexDataPath;
  private static Optional<Path> tempIndexConfigsPath;
  private static Path dataPath;
  private static Path logPath;
  private static Path configPath;
  private static Path workingDirectoryPath;
  private static Path reportDirectoryPath;
  private static Path exampleConfigPath;
  private static Path defaultPath;

  private static StorageService storage;
  private static ModelService model;
  private static IndexService index;
  private static SolrClient solr;
  private static boolean FEATURE_OVERRIDE_INDEX_CONFIGS = true;

  private static boolean TEST_DEPLOY_SOLR = true;
  private static boolean TEST_DEPLOY_LDAP = true;
  private static boolean TEST_DEPLOY_SCANNER = true;
  private static boolean TEST_DEPLOY_ORCHESTRATOR = true;
  private static boolean TEST_DEPLOY_PLUGIN_MANAGER = true;
  private static boolean TEST_DEPLOY_DEFAULT_RESOURCES = true;
  private static SolrType TEST_SOLR_TYPE = null;

  // Metrics related objects
  private static MetricRegistry metricsRegistry;
  private static JmxReporter jmxMetricsReporter;

  // Orchestrator related objects
  private static PluginManager pluginManager;
  // FIXME we should have only "one" orchestrator
  private static PluginOrchestrator pluginOrchestrator;
  private static AkkaDistributedPluginOrchestrator akkaDistributedPluginOrchestrator;
  private static AkkaDistributedPluginWorker akkaDistributedPluginWorker;
  private static boolean FEATURE_DISTRIBUTED_AKKA = false;

  private static LdapUtility ldapUtility;
  private static Path rodaApacheDSDataDirectory = null;

  // TransferredResources related objects
  private static TransferredResourcesScanner transferredResourcesScanner;

  // Configuration related objects
  private static CompositeConfiguration rodaConfiguration = null;
  private static List<String> configurationFiles = null;

  // Caches
  private static CacheLoader<Pair<String, String>, Optional<Schema>> RODA_SCHEMAS_LOADER = new SchemasCacheLoader();
  private static LoadingCache<Pair<String, String>, Optional<Schema>> RODA_SCHEMAS_CACHE = CacheBuilder.newBuilder()
    .build(RODA_SCHEMAS_LOADER);

  private static LoadingCache<Locale, Messages> I18N_CACHE = CacheBuilder.newBuilder()
    .build(new CacheLoader<Locale, Messages>() {

      @Override
      public Messages load(Locale locale) throws Exception {
        return new Messages(locale, getConfigPath().resolve(RodaConstants.CORE_I18N_FOLDER));
      }
    });

  private static Map<String, Map<String, String>> rodaPropertiesCache = null;

  /** Private empty constructor */
  private RodaCoreFactory() {

  }

  public static boolean instantiatedWithoutErrors() {
    return instantiatedWithoutErrors;
  }

  public static void instantiate() {
    NodeType nodeType = NodeType
      .valueOf(getSystemProperty(RodaConstants.CORE_NODE_TYPE, RodaConstants.DEFAULT_NODE_TYPE.name()));

    if (nodeType == RodaConstants.NodeType.MASTER) {
      instantiateMaster();
    } else if (nodeType == RodaConstants.NodeType.TEST) {
      instantiateTest();
    } else if (nodeType == RodaConstants.NodeType.WORKER) {
      instantiateWorker();
    } else {
      LOGGER.error("Unknown node type '{}'", nodeType);
    }
  }

  public static void instantiateMaster() {
    instantiate(NodeType.MASTER);
  }

  public static void instantiateWorker() {
    instantiate(NodeType.WORKER);
  }

  public static void instantiateTest(boolean deploySolr, boolean deployLdap, boolean deployTransferredResourcesScanner,
    boolean deployOrchestrator, boolean deployPluginManager, boolean deployDefaultResources) {
    TEST_DEPLOY_SOLR = deploySolr;
    TEST_DEPLOY_LDAP = deployLdap;
    TEST_DEPLOY_SCANNER = deployTransferredResourcesScanner;
    TEST_DEPLOY_ORCHESTRATOR = deployOrchestrator;
    TEST_DEPLOY_PLUGIN_MANAGER = deployPluginManager;
    TEST_DEPLOY_DEFAULT_RESOURCES = deployDefaultResources;
    instantiated = false;
    instantiate(NodeType.TEST);
  }

  public static void instantiateTest(boolean deploySolr, boolean deployLdap, boolean deployTransferredResourcesScanner,
    boolean deployOrchestrator, boolean deployPluginManager, boolean deployDefaultResources, SolrType solrType) {
    TEST_SOLR_TYPE = solrType;
    instantiateTest(deploySolr, deployLdap, deployTransferredResourcesScanner, deployOrchestrator, deployPluginManager,
      deployDefaultResources);
  }

  public static void instantiateTest() {
    instantiated = false;
    instantiate(NodeType.TEST);
  }

  private static void instantiate(NodeType nodeType) {
    RodaCoreFactory.nodeType = nodeType;

    if (!instantiated) {
      try {
        // determine RODA HOME
        rodaHomePath = determineRodaHomePath();
        LOGGER.debug("RODA HOME is {}", rodaHomePath);

        // instantiate essential directories
        instantiateEssentialDirectories(nodeType);
        LOGGER.debug("Finished instantiating essential directories");

        // load core configurations
        rodaConfiguration = new CompositeConfiguration();
        configurationFiles = new ArrayList<>();
        rodaPropertiesCache = new HashMap<>();
        addConfiguration("roda-core.properties");
        addConfiguration("roda-core-formats.properties");
        LOGGER.debug("Finished loading roda-core.properties & roda-core-formats.properties");
        addConfiguration("roda-roles.properties");
        LOGGER.debug("Finished loading roda-roles.properties");

        // initialize working directory
        initializeWorkingDirectory();

        // initialize reports directory
        initializeReportsDirectory();

        // initialize metrics stuff
        initializeMetrics();

        // instantiate storage and model service
        instantiateStorageAndModel();
        LOGGER.debug("Finished instantiating storage & model");

        // instantiate solr and index service
        instantiateSolrAndIndexService();
        LOGGER.debug("Finished instantiating solr & index");

        instantiateNodeSpecificObjects(nodeType);
        LOGGER.debug("Finished instantiating node specific objects");

        // verify if is necessary to perform a model/index migration
        MigrationManager migrationManager = new MigrationManager(dataPath);
        if (NodeType.MASTER == nodeType
          && migrationManager.isNecessaryToPerformMigration(getSolr(), tempIndexConfigsPath)) {
          // migrationManager.setupModelMigrations();
          // migrationManager.performModelMigrations();
          throw new GenericException("It's necessary to do a model/index migration");
        }

        instantiateDefaultObjects();
        LOGGER.debug("Finished instantiating default objects");

        // instantiate plugin manager
        // 20160920 hsilva: this must be the last thing to be instantiated as
        // problems may araise when instantiating objects at the same time the
        // plugin manager is loading both internal & external plugins (it looks
        // like Reflections is the blame)
        instantiatePluginManager();
        LOGGER.debug("Finished instantiating plugin manager");

        instantiated = true;

      } catch (ConfigurationException e) {
        LOGGER.error("Error loading roda properties", e);
        instantiatedWithoutErrors = false;
      } catch (URISyntaxException e) {
        LOGGER.error("Error instantiating solr/index model", e);
        instantiatedWithoutErrors = false;
      } catch (GenericException e) {
        if (!migrationMode) {
          LOGGER.error("Error instantiating storage model", e);
        }
        instantiatedWithoutErrors = false;
      } catch (Exception e) {
        LOGGER.error("Error instantiating " + RodaCoreFactory.class.getSimpleName(), e);
        instantiatedWithoutErrors = false;
      }

      // last log message that state if system was loaded without errors or not
      LOGGER.info("RODA Core loading completed {}",
        migrationMode ? "(migration mode)"
          : (instantiatedWithoutErrors ? "with success!"
            : "with some errors!!! See logs because these errors might cause instability in the system."));
    }
  }

  private static void initializeWorkingDirectory() {
    try {
      workingDirectoryPath = Paths
        .get(getRodaConfiguration().getString("core.workingdirectory", getSystemProperty("java.io.tmpdir", "tmp")));
      Files.createDirectories(workingDirectoryPath);
    } catch (IOException e) {
      throw new RuntimeException("Unable to create RODA WORKING DIRECTORY " + workingDirectoryPath + ". Aborting...",
        e);
    }

  }

  private static void initializeReportsDirectory() {
    try {
      reportDirectoryPath = getRodaHomePath().resolve(RodaConstants.CORE_REPORT_FOLDER);
      Files.createDirectories(reportDirectoryPath);
    } catch (IOException e) {
      throw new RuntimeException("Unable to create RODA Reports DIRECTORY " + reportDirectoryPath + ". Aborting...", e);
    }
  }

  private static void initializeMetrics() {
    metricsRegistry = new MetricRegistry();
    if (getSystemProperty("com.sun.management.jmxremote", null) != null) {
      jmxMetricsReporter = JmxReporter.forRegistry(metricsRegistry).inDomain("RODA").build();
      jmxMetricsReporter.start();
    }
  }

  private static Path determineRodaHomePath() {
    Path rodaHomePath;
    if (System.getProperty(RodaConstants.INSTALL_FOLDER_SYSTEM_PROPERTY) != null) {
      rodaHomePath = Paths.get(System.getProperty(RodaConstants.INSTALL_FOLDER_SYSTEM_PROPERTY));
    } else if (System.getenv(RodaConstants.INSTALL_FOLDER_ENVIRONEMNT_VARIABLE) != null) {
      rodaHomePath = Paths.get(System.getenv(RodaConstants.INSTALL_FOLDER_ENVIRONEMNT_VARIABLE));
    } else {
      // last attempt (using user home and hidden directory called .roda)
      String userHome = System.getProperty("user.home");
      rodaHomePath = Paths.get(userHome, ".roda");
      if (!FSUtils.exists(rodaHomePath)) {
        try {
          Files.createDirectories(rodaHomePath);
        } catch (IOException e) {
          throw new RuntimeException("Unable to create RODA HOME " + rodaHomePath + ". Aborting...", e);
        }
      }

      // set roda.home in order to correctly configure logging even if no
      // property has been defined
      System.setProperty(RodaConstants.INSTALL_FOLDER_SYSTEM_PROPERTY, rodaHomePath.toString());
    }

    // instantiate essential directories
    configPath = rodaHomePath.resolve(RodaConstants.CORE_CONFIG_FOLDER);
    exampleConfigPath = rodaHomePath.resolve(RodaConstants.CORE_EXAMPLE_CONFIG_FOLDER);
    defaultPath = rodaHomePath.resolve(RodaConstants.CORE_DEFAULT_FOLDER);
    dataPath = rodaHomePath.resolve(RodaConstants.CORE_DATA_FOLDER);
    logPath = dataPath.resolve(RodaConstants.CORE_LOG_FOLDER);
    storagePath = dataPath.resolve(RodaConstants.CORE_STORAGE_FOLDER);
    indexDataPath = dataPath.resolve(RodaConstants.CORE_INDEX_FOLDER);

    // configure logback
    if (nodeType != NodeType.TEST) {
      configureLogback();
    }

    return rodaHomePath;
  }

  private static void configureLogback() {
    try {
      LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
      JoranConfigurator configurator = new JoranConfigurator();
      configurator.setContext(context);
      context.reset();
      configurator.doConfigure(getConfigurationFile("logback.xml"));
    } catch (JoranException e) {
      LOGGER.error("Error configuring logback", e);
    }
  }

  private static void instantiateEssentialDirectories(NodeType nodeType) {
    List<Path> essentialDirectories = new ArrayList<>();
    essentialDirectories.add(configPath);
    essentialDirectories.add(rodaHomePath.resolve(RodaConstants.CORE_LOG_FOLDER));
    essentialDirectories.add(dataPath);
    essentialDirectories.add(logPath);
    essentialDirectories.add(storagePath);
    essentialDirectories.add(indexDataPath);
    essentialDirectories.add(exampleConfigPath);

    for (Path path : essentialDirectories) {
      try {
        if (!FSUtils.exists(path)) {
          Files.createDirectories(path);
        }
      } catch (IOException e) {
        LOGGER.error("Unable to create " + path, e);
        instantiatedWithoutErrors = false;
      }
    }

    if (!nodeType.equals(NodeType.TEST)) {
      // copy configs folder from classpath to example folder
      try {
        FSUtils.deletePathQuietly(exampleConfigPath);
        Files.createDirectories(exampleConfigPath);
        copyFilesFromClasspath(RodaConstants.CORE_CONFIG_FOLDER + "/", exampleConfigPath, true,
          Arrays.asList(RodaConstants.CORE_CONFIG_FOLDER + "/" + RodaConstants.CORE_LDAP_FOLDER,
            RodaConstants.CORE_CONFIG_FOLDER + "/" + RodaConstants.CORE_INDEX_FOLDER,
            RodaConstants.CORE_CONFIG_FOLDER + "/" + RodaConstants.CORE_I18N_FOLDER + "/"
              + RodaConstants.CORE_I18N_CLIENT_FOLDER,
            RodaConstants.CORE_CONFIG_FOLDER + "/" + RodaConstants.CORE_I18N_FOLDER + "/"
              + RodaConstants.CORE_I18_GWT_XML_FILE));
      } catch (IOException e) {
        LOGGER.error("Unable to create " + exampleConfigPath, e);
        instantiatedWithoutErrors = false;
      }
    }

  }

  private static void instantiateDefaultObjects() {
    if (TEST_DEPLOY_DEFAULT_RESOURCES) {
      try {
        CloseableIterable<Resource> resources = storage.listResourcesUnderContainer(DefaultStoragePath.parse(""), true);
        Iterator<Resource> resourceIterator = resources.iterator();
        boolean hasFileResources = false;

        while (resourceIterator.hasNext() && !hasFileResources) {
          Resource resource = resourceIterator.next();
          if (!resource.isDirectory()) {
            hasFileResources = true;
          }
        }
        IOUtils.closeQuietly(resources);

        if (!hasFileResources) {
          copyFilesFromClasspath(RodaConstants.CORE_DEFAULT_FOLDER + "/", rodaHomePath, true);

          // 20160712 hsilva: it needs to be this way as the resources are
          // copied to the file system and storage can be of a different type
          // (e.g. fedora)
          FileStorageService fileStorageService = new FileStorageService(storagePath);

          index.reindexRisks(fileStorageService);
          index.reindexFormats(fileStorageService);
          index.reindexAIPs();
          // reindex other default objects HERE
        }
      } catch (AuthorizationDeniedException | RequestNotValidException | NotFoundException | GenericException e) {
        LOGGER.error("Cannot load default objects", e);
      }
    }
  }

  private static void copyFilesFromClasspath(String classpathPrefix, Path destinationDirectory,
    boolean removeClasspathPrefixFromFinalPath) {
    copyFilesFromClasspath(classpathPrefix, destinationDirectory, removeClasspathPrefixFromFinalPath,
      Collections.emptyList());
  }

  private static void copyFilesFromClasspath(String classpathPrefix, Path destinationDirectory,
    boolean removeClasspathPrefixFromFinalPath, List<String> excludePaths) {

    List<ClassLoader> classLoadersList = new LinkedList<>();
    classLoadersList.add(ClasspathHelper.contextClassLoader());

    Reflections reflections = new Reflections(
      new ConfigurationBuilder().filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix(classpathPrefix)))
        .setScanners(new ResourcesScanner())
        .setUrls(ClasspathHelper.forClassLoader(classLoadersList.toArray(new ClassLoader[] {}))));

    Set<String> resources = reflections.getResources(Pattern.compile(".*"));

    LOGGER.info("Copy files from classpath prefix={}, destination={}, removePrefix={}, excludePaths={}, #resources={}",
      classpathPrefix, destinationDirectory, removeClasspathPrefixFromFinalPath, excludePaths, resources.size());

    for (String resource : resources) {
      boolean exclude = false;
      for (String excludePath : excludePaths) {
        if (resource.startsWith(excludePath)) {
          exclude = true;
          break;
        }
      }
      if (exclude) {
        continue;
      }

      InputStream originStream = RodaCoreFactory.class.getClassLoader().getResourceAsStream(resource);
      Path destinyPath;

      String resourceFileName = resource;

      // Removing ":" escape
      resourceFileName = resourceFileName.replace("::", ":");

      if (removeClasspathPrefixFromFinalPath) {
        destinyPath = destinationDirectory.resolve(resourceFileName.replaceFirst(classpathPrefix, ""));
      } else {
        destinyPath = destinationDirectory.resolve(resourceFileName);
      }

      try {
        // create all parent directories
        Files.createDirectories(destinyPath.getParent());
        // copy file
        Files.copy(originStream, destinyPath, StandardCopyOption.REPLACE_EXISTING);
      } catch (IOException e) {
        LOGGER.error("Error copying file from classpath: {} to {} (reason: {})", originStream, destinyPath,
          e.getMessage());
        instantiatedWithoutErrors = false;
      } finally {
        RodaUtils.closeQuietly(originStream);
      }
    }
  }

  private static void copyFilesFromClasspath(String classpathPrefix, Path destinationDirectory) {
    copyFilesFromClasspath(classpathPrefix, destinationDirectory, false);
  }

  private static void instantiatePluginManager() {
    if (nodeType == NodeType.MASTER || nodeType == NodeType.WORKER || TEST_DEPLOY_PLUGIN_MANAGER) {
      try {
        pluginManager = PluginManager.instantiatePluginManager(getConfigPath(), getPluginsPath());
      } catch (PluginManagerException e) {
        LOGGER.error("Error instantiating PluginManager", e);
        instantiatedWithoutErrors = false;
      }
    }
  }

  private static void instantiateStorageAndModel() throws GenericException {
    storage = instantiateStorage();
    LOGGER.debug("Finished instantiating storage...");
    model = new ModelService(storage);
    LOGGER.debug("Finished instantiating model...");
  }

  private static StorageService instantiateStorage() throws GenericException {
    StorageType storageType = StorageType.valueOf(
      getRodaConfiguration().getString(RodaConstants.CORE_STORAGE_TYPE, RodaConstants.DEFAULT_STORAGE_TYPE.toString()));
    if (storageType == RodaConstants.StorageType.FEDORA4) {
      String url = getRodaConfiguration().getString(RodaConstants.CORE_STORAGE_FEDORA4_URL,
        "http://localhost:8983/solr/");
      String username = getRodaConfiguration().getString(RodaConstants.CORE_STORAGE_FEDORA4_USERNAME, "");
      String password = getRodaConfiguration().getString(RodaConstants.CORE_STORAGE_FEDORA4_PASSWORD, "");

      if (StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)) {
        LOGGER.debug("Going to instantiate Fedora with url '{}' username '{}' & password '{}'", url, username,
          password);
        return new FedoraStorageService(url, username, password);
      } else {
        LOGGER.debug("Going to instantiate Fedora with url '{}'", url);
        return new FedoraStorageService(url);
      }
    } else if (storageType == RodaConstants.StorageType.FILESYSTEM) {
      LOGGER.debug("Going to instantiate Filesystem on '{}'", storagePath);
      String trashDirName = getRodaConfiguration().getString("core.storage.filesystem.trash", "trash");
      return new FileStorageService(storagePath, trashDirName);
    } else {
      LOGGER.error("Unknown storage service '{}'", storageType.name());
      throw new GenericException();
    }
  }

  /**
   * <p>
   * Warnings like
   * </p>
   * <code>2016-03-21 11:21:34,319 WARN  org.apache.solr.core.Config - Beginning with Solr 5.5, <maxMergeDocs> is deprecated, configure it on the relevant <mergePolicyFactory> instead.</code>
   * <br/>
   * <code>2016-03-21 11:21:34,327 WARN  org.apache.solr.core.Config - Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.</code>
   * <p>
   * are due to a bug, explained in
   * https://issues.apache.org/jira/browse/SOLR-8734, as we don't declare those
   * parameters in RODA solr configurations. The warning will be removed and as
   * soon as that happens, this messages should be deleted as well.
   * </p>
   * 
   */
  private static void instantiateSolrAndIndexService() throws URISyntaxException {
    if (nodeType == NodeType.MASTER) {
      tempIndexConfigsPath = Optional.empty();
      Path solrHome = configPath.resolve(RodaConstants.CORE_INDEX_FOLDER);
      if (!FSUtils.exists(solrHome) || FEATURE_OVERRIDE_INDEX_CONFIGS) {
        try {
          Path tempConfig = Files.createTempDirectory(getWorkingDirectory(), RodaConstants.CORE_INDEX_FOLDER);
          toDeleteDuringShutdown.add(tempConfig);
          tempIndexConfigsPath = Optional.of(tempConfig);
          copyFilesFromClasspath(RodaConstants.CORE_CONFIG_FOLDER + "/" + RodaConstants.CORE_INDEX_FOLDER + "/",
            tempConfig);
          solrHome = tempConfig.resolve(RodaConstants.CORE_CONFIG_FOLDER).resolve(RodaConstants.CORE_INDEX_FOLDER);
          LOGGER.info("Using SOLR home: {}", solrHome);
        } catch (IOException e) {
          LOGGER.error("Error creating temporary SOLR home", e);
          instantiatedWithoutErrors = false;
        }
      }

      // instantiate solr
      solr = instantiateSolr(solrHome);

      // instantiate index related object
      index = new IndexService(solr, model);
    } else if (nodeType == NodeType.TEST && TEST_DEPLOY_SOLR) {
      try {
        Path tempConfig = Files.createTempDirectory(getWorkingDirectory(), RodaConstants.CORE_INDEX_FOLDER);
        copyFilesFromClasspath(RodaConstants.CORE_CONFIG_FOLDER + "/" + RodaConstants.CORE_INDEX_FOLDER + "/",
          tempConfig, true);

        // instantiate solr
        solr = instantiateSolr(tempConfig);

        // instantiate index related object
        index = new IndexService(solr, model);
      } catch (IOException e) {
        LOGGER.error("Unable to instantiate Solr in TEST mode", e);
        instantiatedWithoutErrors = false;
      }
    }
  }

  private static SolrClient instantiateSolr(Path solrHome) {
    SolrType solrType = SolrType.valueOf(
      getRodaConfiguration().getString(RodaConstants.CORE_SOLR_TYPE, RodaConstants.DEFAULT_SOLR_TYPE.toString()));
    if (TEST_SOLR_TYPE != null) {
      solrType = TEST_SOLR_TYPE;
    }

    if (solrType == RodaConstants.SolrType.HTTP) {
      String solrBaseUrl = getRodaConfiguration().getString(RodaConstants.CORE_SOLR_HTTP_URL,
        "http://localhost:8983/solr/");
      return new HttpSolrClient(solrBaseUrl);
    } else if (solrType == RodaConstants.SolrType.HTTP_CLOUD) {
      String solrCloudZooKeeperUrls = getRodaConfiguration().getString(RodaConstants.CORE_SOLR_HTTP_CLOUD_URLS,
        "localhost:2181,localhost:2182,localhost:2183");
      return new CloudSolrClient(solrCloudZooKeeperUrls);
    } else {
      // default to Embedded
      setSolrSystemProperties();
      return new EmbeddedSolrServer(solrHome, "test");
    }
  }

  private static void setSolrSystemProperties() {
    System.setProperty("solr.data.dir", indexDataPath.toString());
    System.setProperty("solr.data.dir.aip", indexDataPath.resolve(RodaConstants.CORE_AIP_FOLDER).toString());
    System.setProperty("solr.data.dir.representations",
      indexDataPath.resolve(RodaConstants.CORE_REPRESENTATION_FOLDER).toString());
    System.setProperty("solr.data.dir.file", indexDataPath.resolve(RodaConstants.CORE_FILE_FOLDER).toString());
    System.setProperty("solr.data.dir.preservationevent",
      indexDataPath.resolve(RodaConstants.CORE_PRESERVATIONEVENT_FOLDER).toString());
    System.setProperty("solr.data.dir.preservationagent",
      indexDataPath.resolve(RodaConstants.CORE_PRESERVATIONAGENT_FOLDER).toString());
    System.setProperty("solr.data.dir.actionlog",
      indexDataPath.resolve(RodaConstants.CORE_ACTIONLOG_FOLDER).toString());
    System.setProperty("solr.data.dir.members", indexDataPath.resolve(RodaConstants.CORE_MEMBERS_FOLDER).toString());
    System.setProperty("solr.data.dir.transferredresource",
      indexDataPath.resolve(RodaConstants.CORE_TRANSFERREDRESOURCE_FOLDER).toString());
    System.setProperty("solr.data.dir.job", indexDataPath.resolve(RodaConstants.CORE_JOB_FOLDER).toString());
    System.setProperty("solr.data.dir.jobreport",
      indexDataPath.resolve(RodaConstants.CORE_JOBREPORT_FOLDER).toString());
    System.setProperty("solr.data.dir.risk", indexDataPath.resolve(RodaConstants.CORE_RISK_FOLDER).toString());
    System.setProperty("solr.data.dir.agent", indexDataPath.resolve(RodaConstants.CORE_AGENT_FOLDER).toString());
    System.setProperty("solr.data.dir.format", indexDataPath.resolve(RodaConstants.CORE_FORMAT_FOLDER).toString());
    System.setProperty("solr.data.dir.notification",
      indexDataPath.resolve(RodaConstants.CORE_NOTIFICATION_FOLDER).toString());
    System.setProperty("solr.data.dir.riskincidence",
      indexDataPath.resolve(RodaConstants.CORE_RISKINCIDENCE_FOLDER).toString());
    System.setProperty("solr.data.dir.dip", indexDataPath.resolve(RodaConstants.CORE_DIP_FOLDER).toString());
    System.setProperty("solr.data.dir.dipfile", indexDataPath.resolve(RodaConstants.CORE_DIP_FILE_FOLDER).toString());
  }

  private static void instantiateNodeSpecificObjects(NodeType nodeType) {
    if (nodeType == NodeType.MASTER) {
      instantiateMasterNodeSpecificObjects();
    } else if (nodeType == NodeType.WORKER) {
      instantiateWorkerNodeSpecificObjects();
    } else if (nodeType == NodeType.TEST) {
      instantiateTestNodeSpecificObjects();
    } else {
      LOGGER.error("Unknown node type '{}'", nodeType);
      instantiatedWithoutErrors = false;
    }
  }

  private static void instantiateMasterNodeSpecificObjects() {
    if (FEATURE_DISTRIBUTED_AKKA) {
      akkaDistributedPluginOrchestrator = new AkkaDistributedPluginOrchestrator(
        getSystemProperty(RodaConstants.CORE_NODE_HOSTNAME, RodaConstants.DEFAULT_NODE_HOSTNAME),
        getSystemProperty(RodaConstants.CORE_NODE_PORT, RodaConstants.DEFAULT_NODE_PORT));
      akkaDistributedPluginOrchestrator.cleanUnfinishedJobs();
    } else {
      // pluginOrchestrator = new EmbeddedActionOrchestrator();
      pluginOrchestrator = new AkkaEmbeddedPluginOrchestrator();
      pluginOrchestrator.cleanUnfinishedJobs();
    }

    startApacheDS();

    instantiateTransferredResourcesScanner();

    processPreservationEventTypeProperties();
  }

  private static void instantiateWorkerNodeSpecificObjects() {
    akkaDistributedPluginWorker = new AkkaDistributedPluginWorker(
      getSystemProperty(RodaConstants.CORE_CLUSTER_HOSTNAME, RodaConstants.DEFAULT_NODE_HOSTNAME),
      getSystemProperty(RodaConstants.CORE_CLUSTER_PORT, RodaConstants.DEFAULT_NODE_PORT),
      getSystemProperty(RodaConstants.CORE_NODE_HOSTNAME, RodaConstants.DEFAULT_NODE_HOSTNAME),
      getSystemProperty(RodaConstants.CORE_NODE_PORT, "0"));
  }

  private static void instantiateTestNodeSpecificObjects() {
    if (TEST_DEPLOY_LDAP) {
      startApacheDS();
    }

    if (TEST_DEPLOY_SCANNER) {
      instantiateTransferredResourcesScanner();
    }

    if (TEST_DEPLOY_ORCHESTRATOR) {
      pluginOrchestrator = new AkkaEmbeddedPluginOrchestrator();
    }
  }

  public static void addLogger(String loggerConfigurationFile) {
    URL loggerConfigurationFileUrl = getConfigurationFile(loggerConfigurationFile);
    if (loggerConfigurationFileUrl != null) {
      System.setProperty("roda.logback.include", loggerConfigurationFileUrl.toString());
      configureLogback();
    }
  }

  private static String getSystemProperty(String property, String defaultValue) {
    String ret = defaultValue;
    if (System.getProperty(property) != null) {
      ret = System.getProperty(property);
    }
    return ret;
  }

  public static void shutdown() throws IOException {
    if (instantiated) {

      if (nodeType == NodeType.MASTER) {
        solr.close();
        stopApacheDS();
        pluginManager.shutdown();
        pluginOrchestrator.shutdown();
      } else if (nodeType == NodeType.WORKER) {
        pluginManager.shutdown();
      } else if (nodeType == NodeType.TEST) {
        if (TEST_DEPLOY_SOLR) {
          solr.close();
        }
        if (TEST_DEPLOY_LDAP) {
          stopApacheDS();
        }
        if (TEST_DEPLOY_ORCHESTRATOR) {
          pluginOrchestrator.shutdown();
        }
        // final cleanup
        FSUtils.deletePathQuietly(workingDirectoryPath);
      }

      // stop jmx metrics reporter
      if (getSystemProperty("com.sun.management.jmxremote", null) != null) {
        jmxMetricsReporter.stop();
      }

      // delete resources that are no longer needed
      toDeleteDuringShutdown.forEach(e -> FSUtils.deletePathQuietly(e));

    }
  }

  public static MetricRegistry getMetrics() {
    return metricsRegistry;
  }

  /**
   * Start ApacheDS.
   */
  private static void startApacheDS() {
    rodaApacheDSDataDirectory = RodaCoreFactory.getDataPath().resolve(RodaConstants.CORE_LDAP_FOLDER);

    try {
      final Configuration rodaConfig = RodaCoreFactory.getRodaConfiguration();

      final boolean ldapStartServer = rodaConfig.getBoolean("ldap.startServer", false);
      final int ldapPort = rodaConfig.getInt("ldap.port", RodaConstants.CORE_LDAP_DEFAULT_PORT);
      final String ldapBaseDN = rodaConfig.getString("ldap.baseDN", "dc=roda,dc=org");
      final String ldapPeopleDN = rodaConfig.getString("ldap.peopleDN", "ou=users,dc=roda,dc=org");
      final String ldapGroupsDN = rodaConfig.getString("ldap.groupsDN", "ou=groups,dc=roda,dc=org");
      final String ldapRolesDN = rodaConfig.getString("ldap.rolesDN", "ou=groups,dc=roda,dc=org");
      final String ldapAdminDN = rodaConfig.getString("ldap.adminDN", "ou=groups,dc=roda,dc=org");
      final String ldapAdminPassword = rodaConfig.getString("ldap.adminPassword", "roda");
      final String ldapPasswordDigestAlgorithm = rodaConfig.getString("ldap.passwordDigestAlgorithm", "MD5");
      final List<String> ldapProtectedUsers = RodaUtils.copyList(rodaConfig.getList("ldap.protectedUsers"));
      final List<String> ldapProtectedGroups = RodaUtils.copyList(rodaConfig.getList("ldap.protectedGroups"));
      final String rodaGuestDN = rodaConfig.getString("ldap.rodaGuestDN", "uid=guest,ou=users,dc=roda,dc=org");
      final String rodaAdminDN = rodaConfig.getString("ldap.rodaAdminDN", "uid=admin,ou=users,dc=roda,dc=org");
      final String rodaAdministratorsDN = rodaConfig.getString("ldap.rodaAdministratorsDN",
        "cn=administrators,ou=groups,dc=roda,dc=org");

      RodaCoreFactory.ldapUtility = new LdapUtility(ldapStartServer, ldapPort, ldapBaseDN, ldapPeopleDN, ldapGroupsDN,
        ldapRolesDN, ldapAdminDN, ldapAdminPassword, ldapPasswordDigestAlgorithm, ldapProtectedUsers,
        ldapProtectedGroups, rodaGuestDN, rodaAdminDN, rodaApacheDSDataDirectory);
      ldapUtility.setRODAAdministratorsDN(rodaAdministratorsDN);

      UserUtility.setLdapUtility(ldapUtility);

      if (!FSUtils.exists(rodaApacheDSDataDirectory)) {
        Files.createDirectories(rodaApacheDSDataDirectory);
        final List<String> ldifFileNames = Arrays.asList("users.ldif", "groups.ldif", "roles.ldif");
        final List<String> ldifs = new ArrayList<>();
        for (String ldifFileName : ldifFileNames) {
          final InputStream ldifInputStream = RodaCoreFactory
            .getConfigurationFileAsStream(RodaConstants.CORE_LDAP_FOLDER + "/" + ldifFileName);
          ldifs.add(IOUtils.toString(ldifInputStream, RodaConstants.DEFAULT_ENCODING));
          RodaUtils.closeQuietly(ldifInputStream);
        }

        RodaCoreFactory.ldapUtility.initDirectoryService(ldifs);
        indexUsersAndGroupsFromLDAP();
      } else {
        RodaCoreFactory.ldapUtility.initDirectoryService();
      }

      createRoles(rodaConfig);
      indexUsersAndGroupsFromLDAP();

    } catch (final Exception e) {
      LOGGER.error("Error starting up embedded ApacheDS", e);
      instantiatedWithoutErrors = false;
    }

  }

  private static void stopApacheDS() {
    try {
      RodaCoreFactory.ldapUtility.stopService();
    } catch (final Exception e) {
      LOGGER.error("Error while shutting down ApacheDS embedded server", e);
    }
  }

  /**
   * For each role in roda-roles.properties create the role in LDAP if it don't
   * exist already.
   * 
   * @param rodaConfig
   *          roda configuration
   * @throws GenericException
   *           if something unexpected happens creating roles.
   */
  private static void createRoles(final Configuration rodaConfig) throws GenericException {
    final Iterator<String> keys = rodaConfig.getKeys("core.roles");
    final Set<String> roles = new HashSet<>();
    while (keys.hasNext()) {
      roles.addAll(Arrays.asList(rodaConfig.getStringArray(keys.next())));
    }
    for (final String role : roles) {
      try {
        if (StringUtils.isNotBlank(role)) {
          RodaCoreFactory.ldapUtility.addRole(role);
          LOGGER.debug("Created LDAP role {}", role);
        }
      } catch (final RoleAlreadyExistsException e) {
        LOGGER.debug("Role {} already exists.", role);
        LOGGER.trace(e.getMessage(), e);
      }
    }
  }

  private static void indexUsersAndGroupsFromLDAP()
    throws GenericException, IllegalOperationException, NotFoundException, AlreadyExistsException {
    for (User user : model.listUsers()) {
      LOGGER.debug("User to be indexed: {}", user);
      model.notifyUserUpdated(user);
    }
    for (Group group : model.listGroups()) {
      LOGGER.debug("Group to be indexed: {}", group);
      model.notifyGroupUpdated(group);
    }
  }

  public static void instantiateTransferredResourcesScanner() {
    try {
      String transferredResourcesFolder = getRodaConfiguration().getString("transferredResources.folder",
        RodaConstants.CORE_TRANSFERREDRESOURCE_FOLDER);
      Path transferredResourcesFolderPath = dataPath.resolve(transferredResourcesFolder);
      if (!FSUtils.exists(transferredResourcesFolderPath)) {
        Files.createDirectories(transferredResourcesFolderPath);
      }

      transferredResourcesScanner = new TransferredResourcesScanner(transferredResourcesFolderPath, getIndexService());
    } catch (final Exception e) {
      LOGGER.error("Error starting Transferred Resources Scanner: " + e.getMessage(), e);
      instantiatedWithoutErrors = false;
    }
  }

  public static boolean getTransferredResourcesScannerUpdateStatus(Optional<String> folderRelativePath) {
    return TransferUpdateStatus.getInstance().isUpdatingStatus(folderRelativePath);
  }

  public static void setTransferredResourcesScannerUpdateStatus(Optional<String> folderRelativePath,
    boolean isUpdating) {
    TransferUpdateStatus.getInstance().setUpdatingStatus(folderRelativePath, isUpdating);
  }

  public static StorageService getStorageService() {
    return storage;
  }

  public static ModelService getModelService() {
    return model;
  }

  public static IndexService getIndexService() {
    return index;
  }

  public static SolrClient getSolr() {
    return solr;
  }

  public static void setSolr(SolrClient solr) {
    RodaCoreFactory.solr = solr;
  }

  public static PluginManager getPluginManager() {
    return PluginManager.getInstance();
  }

  public static PluginOrchestrator getPluginOrchestrator() {
    return pluginOrchestrator;
  }

  public static AkkaDistributedPluginOrchestrator getAkkaDistributedPluginOrchestrator() {
    return akkaDistributedPluginOrchestrator;
  }

  public static TransferredResourcesScanner getTransferredResourcesScanner() {
    return transferredResourcesScanner;
  }

  public static NodeType getNodeType() {
    return nodeType;
  }

  public static Path getRodaHomePath() {
    return rodaHomePath;
  }

  public static Path getConfigPath() {
    return configPath;
  }

  public static Path getDefaultPath() {
    return defaultPath;
  }

  public static Path getWorkingDirectory() {
    return workingDirectoryPath;
  }

  public static Path getReportsDirectory() {
    return reportDirectoryPath;
  }

  public static Path getDataPath() {
    return dataPath;
  }

  public static Path getStoragePath() {
    return storagePath;
  }

  public static Path getLogPath() {
    return logPath;
  }

  public static Path getPluginsPath() {
    return configPath.resolve(RodaConstants.CORE_PLUGINS_FOLDER);
  }

  public static void closeSolrServer() {
    try {
      solr.close();
    } catch (IOException e) {
      LOGGER.error("Error while shutting down solr", e);
    }
  }

  /*
   * Configuration related functionalities
   */
  public static void addConfiguration(String configurationFile) throws ConfigurationException {
    Configuration configuration = getConfiguration(configurationFile);
    rodaConfiguration.addConfiguration(configuration);
    configurationFiles.add(configurationFile);
  }

  public static Configuration getConfiguration(String configurationFile) throws ConfigurationException {
    Path config = RodaCoreFactory.getConfigPath().resolve(configurationFile);
    PropertiesConfiguration propertiesConfiguration = new PropertiesConfiguration();
    propertiesConfiguration.setDelimiterParsingDisabled(true);
    propertiesConfiguration.setEncoding(RodaConstants.DEFAULT_ENCODING);

    if (FSUtils.exists(config)) {
      LOGGER.trace("Loading configuration from file {}", config);
      propertiesConfiguration.load(config.toFile());
      RodaPropertiesReloadStrategy rodaPropertiesReloadStrategy = new RodaPropertiesReloadStrategy();
      rodaPropertiesReloadStrategy.setRefreshDelay(5000);
      propertiesConfiguration.setReloadingStrategy(rodaPropertiesReloadStrategy);
    } else {
      InputStream inputStream = RodaCoreFactory.class
        .getResourceAsStream("/" + RodaConstants.CORE_CONFIG_FOLDER + "/" + configurationFile);
      if (inputStream != null) {
        LOGGER.trace("Loading configuration from classpath {}", configurationFile);
        propertiesConfiguration.load(inputStream);
      } else {
        LOGGER.trace("Configuration {} doesn't exist", configurationFile);
      }
    }

    return propertiesConfiguration;
  }

  public static URL getConfigurationFile(String configurationFile) {
    Path config = RodaCoreFactory.getConfigPath().resolve(configurationFile);
    URL configUri;
    if (FSUtils.exists(config) && !FSUtils.isDirectory(config)
      && config.toAbsolutePath().startsWith(getConfigPath().toAbsolutePath().toString())) {
      try {
        configUri = config.toUri().toURL();
      } catch (MalformedURLException e) {
        LOGGER.error("Configuration {} doesn't exist", configurationFile);
        configUri = null;
      }
    } else {
      URL resource = RodaCoreFactory.class
        .getResource("/" + RodaConstants.CORE_CONFIG_FOLDER + "/" + configurationFile);
      if (resource != null) {
        configUri = resource;
      } else {
        LOGGER.error("Configuration {} doesn't exist", configurationFile);
        configUri = null;
      }
    }

    return configUri;
  }

  public static InputStream getConfigurationFileAsStream(String configurationFile) {
    Path config = getConfigPath().resolve(configurationFile);
    InputStream inputStream = null;
    try {
      if (FSUtils.exists(config) && !FSUtils.isDirectory(config)
        && config.toAbsolutePath().startsWith(getConfigPath().toAbsolutePath().toString())) {
        inputStream = Files.newInputStream(config);
        LOGGER.trace("Loading configuration from file {}", config);
      }
    } catch (IOException e) {
      // do nothing
    }
    if (inputStream == null) {
      inputStream = RodaCoreFactory.class
        .getResourceAsStream("/" + RodaConstants.CORE_CONFIG_FOLDER + "/" + configurationFile);
      LOGGER.trace("Loading configuration from classpath {}", configurationFile);
    }
    return inputStream;
  }

  public static InputStream getConfigurationFileAsStream(String configurationFile, String fallbackConfigurationFile) {
    InputStream inputStream = getConfigurationFileAsStream(configurationFile);
    if (inputStream == null) {
      inputStream = getConfigurationFileAsStream(fallbackConfigurationFile);
    }
    return inputStream;
  }

  public static InputStream getDefaultFileAsStream(String defaultFile, ClassLoader... extraClassLoaders) {
    Path defaultPath = getDefaultPath().resolve(defaultFile);
    InputStream inputStream = null;
    try {
      if (FSUtils.exists(defaultPath) && !FSUtils.isDirectory(defaultPath)
        && defaultPath.toAbsolutePath().startsWith(getDefaultPath().toAbsolutePath().toString())) {
        inputStream = Files.newInputStream(defaultPath);
        LOGGER.debug("Trying to load default from file {}", defaultPath);
      }
    } catch (IOException e) {
      // do nothing
    }

    if (inputStream == null) {
      String fileClassPath = "/" + RodaConstants.CORE_DEFAULT_FOLDER + "/" + defaultFile;
      inputStream = RodaCoreFactory.class.getResourceAsStream(fileClassPath);
      LOGGER.debug("Trying to load default file from classpath {}", fileClassPath);
    }

    if (inputStream == null) {
      String fileClassPath = RodaConstants.CORE_DEFAULT_FOLDER + "/" + defaultFile;
      for (ClassLoader classLoader : extraClassLoaders) {
        LOGGER.debug("Trying to load default file from extra class loader {}", fileClassPath);
        inputStream = classLoader.getResourceAsStream(fileClassPath);
        if (inputStream != null) {
          break;
        }
      }
    }

    return inputStream;
  }

  public static void clearRodaCachableObjectsAfterConfigurationChange() {
    rodaPropertiesCache.clear();
    RODA_SCHEMAS_CACHE.invalidateAll();
    I18N_CACHE.invalidateAll();
    processPreservationEventTypeProperties();

    LOGGER.info("Reloaded roda configurations after file change!");
  }

  private static void processPreservationEventTypeProperties() {
    String prefix = "core.preservation_event_type";

    for (PreservationEventType preservationEventType : PreservationEventType.values()) {
      String value = getRodaConfigurationAsString(prefix, preservationEventType.name());
      if (StringUtils.isNotBlank(value)) {
        preservationEventType.setText(value);
      } else {
        preservationEventType.setText(preservationEventType.getOriginalText());
      }
    }
  }

  public static Optional<Schema> getRodaSchema(String metadataType, String metadataVersion) {
    Optional<Schema> schema = Optional.empty();
    try {
      schema = RODA_SCHEMAS_CACHE.get(Pair.of(metadataType, metadataVersion));
    } catch (ExecutionException e) {
      if (StringUtils.isNotBlank(metadataType)) {
        try {
          schema = RODA_SCHEMAS_CACHE.get(Pair.of(metadataType, null));
        } catch (ExecutionException e2) {
          // Do nothing
        }
      }
    }
    return schema;
  }

  public static Configuration getRodaConfiguration() {
    return rodaConfiguration;
  }

  public static String getConfigurationKey(String... keyParts) {
    StringBuilder sb = new StringBuilder();
    for (String part : keyParts) {
      if (sb.length() != 0) {
        sb.append('.');
      }
      sb.append(part);
    }
    return sb.toString();
  }

  public static String getRodaConfigurationAsString(String... keyParts) {
    return rodaConfiguration.getString(getConfigurationKey(keyParts));
  }

  public static int getRodaConfigurationAsInt(int defaultValue, String... keyParts) {
    return rodaConfiguration.getInt(getConfigurationKey(keyParts), defaultValue);
  }

  public static List<String> getRodaConfigurationAsList(String... keyParts) {
    String[] array = rodaConfiguration.getStringArray(getConfigurationKey(keyParts));
    return Arrays.asList(array).stream().filter(v -> StringUtils.isNotBlank(v)).collect(Collectors.toList());
  }

  public static int getRodaConfigurationAsInt(String... keyParts) {
    return getRodaConfigurationAsInt(0, keyParts);
  }

  public static List<String> getFixityAlgorithms() {
    List<String> algorithms = RodaCoreFactory.getRodaConfigurationAsList("core", "premis", "fixity", "algorithms");
    if (algorithms == null || algorithms.isEmpty()) {
      algorithms = RodaConstants.DEFAULT_ALGORITHMS;
    }
    return algorithms;
  }

  public static Set<String> getFilenamesInsideConfigFolder(String folder) throws IOException {

    Set<String> fileNames = new HashSet<>();

    // get from external config
    Set<String> externalFileNames = new HashSet<>();
    Path configPath = RodaCoreFactory.getConfigPath().resolve(folder);
    Files.walkFileTree(configPath, new SimpleFileVisitor<Path>() {
      @Override
      public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
        externalFileNames.add(file.getFileName().toString());
        return FileVisitResult.CONTINUE;
      }
    });

    fileNames.addAll(externalFileNames);

    // get from internal config
    List<ClassLoader> classLoadersList = new LinkedList<>();
    classLoadersList.add(ClasspathHelper.contextClassLoader());
    Set<String> internalFilesPath = new Reflections(new ConfigurationBuilder()
      .filterInputsBy(
        new FilterBuilder().include(FilterBuilder.prefix("" + RodaConstants.CORE_CONFIG_FOLDER + "/" + folder)))
      .setScanners(new ResourcesScanner())
      .setUrls(ClasspathHelper.forClassLoader(classLoadersList.toArray(new ClassLoader[0]))))
        .getResources(Pattern.compile(".*"));
    for (String internalFilePath : internalFilesPath) {
      fileNames.add(Paths.get(internalFilePath).getFileName().toString());
    }

    return fileNames;
  }

  public static Map<String, String> getPropertiesFromCache(String cacheName, List<String> prefixesToCache) {
    if (rodaPropertiesCache.get(cacheName) == null) {
      fillInPropertiesToCache(cacheName, prefixesToCache);
    }
    return rodaPropertiesCache.get(cacheName);
  }

  private static void fillInPropertiesToCache(String cacheName, List<String> prefixesToCache) {
    if (rodaPropertiesCache.get(cacheName) == null) {
      HashMap<String, String> newCacheEntry = new HashMap<>();

      Configuration configuration = RodaCoreFactory.getRodaConfiguration();
      Iterator<String> keys = configuration.getKeys();
      while (keys.hasNext()) {
        String key = String.class.cast(keys.next());
        String value = configuration.getString(key, "");
        for (String prefixToCache : prefixesToCache) {
          if (key.startsWith(prefixToCache)) {
            newCacheEntry.put(key, value);
            break;
          }
        }
      }
      rodaPropertiesCache.put(cacheName, newCacheEntry);
    }
  }

  public static Messages getI18NMessages(Locale locale) {
    checkForChangesInI18N();
    try {
      return I18N_CACHE.get(locale);
    } catch (ExecutionException e) {
      LOGGER.debug("Could not load messages", e);
      return null;
    }
  }

  private static void checkForChangesInI18N() {
    // i18n is cached and that cache is re-done when changes occur to
    // roda-*.properties (for convenience)
    getRodaConfiguration().getString("");
  }

  /*
   * Command-line accessible functionalities
   */

  public static void runReindex(List<String> args) {
    String entity = args.get(2);
    if (StringUtils.isNotBlank(entity)) {
      if ("users_and_groups".equalsIgnoreCase(entity)) {
        try {
          indexUsersAndGroupsFromLDAP();
        } catch (IllegalOperationException | GenericException | NotFoundException | AlreadyExistsException e) {
          LOGGER.error("Unable to reindex users & groups from LDAP.", e);
        }
      }
    }
  }

  private static void runSolrQuery(List<String> args) {
    String collection = args.get(2);
    String solrQueryString = args.get(3);
    try {
      QueryResponse executeSolrQuery = SolrUtils.executeSolrQuery(solr, collection, solrQueryString);
      SolrDocumentList results = executeSolrQuery.getResults();
      System.out.println("Size: " + results.getNumFound() + "; Returned: " + results.size());
      for (SolrDocument solrDocument : results) {
        System.out.println(">" + solrDocument);
      }
    } catch (SolrServerException | IOException e) {
      e.printStackTrace(System.err);
    }
  }

  private static void printIndexMembers(List<String> args, Filter filter, Sorter sorter, Sublist sublist, Facets facets)
    throws GenericException, RequestNotValidException {
    System.out.println("Index list " + args.get(2));
    IndexResult<RODAMember> users = index.find(RODAMember.class, filter, sorter, sublist, facets, new ArrayList<>());
    for (RODAMember rodaMember : users.getResults()) {
      System.out.println("\t" + rodaMember);
    }
  }

  private static void printCountSips(Sorter sorter, Sublist sublist, Facets facets)
    throws GenericException, RequestNotValidException {
    Filter filter = new Filter(new SimpleFilterParameter(RodaConstants.TRANSFERRED_RESOURCE_ISFILE, "true"));
    long countFiles = index.count(TransferredResource.class, filter);
    filter = new Filter(new SimpleFilterParameter(RodaConstants.TRANSFERRED_RESOURCE_ISFILE, "false"));
    long countDirectories = index.count(TransferredResource.class, filter);
    Filter f1 = new Filter();
    FilterParameter p1 = new EmptyKeyFilterParameter(RodaConstants.TRANSFERRED_RESOURCE_PARENT_ID);
    FilterParameter p2 = new BasicSearchFilterParameter(RodaConstants.TRANSFERRED_RESOURCE_ISFILE, "false");
    f1.add(p1);
    f1.add(p2);
    long countSIP = index.count(TransferredResource.class, f1);

    System.out.println("Total number of directories: " + countDirectories);
    System.out.println("Total number of files: " + countFiles);
    System.out.println("Total number of SIPs: " + countSIP);
  }

  private static void printFiles(Sorter sorter, Sublist sublist) throws GenericException, RequestNotValidException {
    Filter filter = new Filter(new SimpleFilterParameter(RodaConstants.FILE_SEARCH, "OLA-OLÁ-1234-XXXX_K"));
    IndexResult<IndexedFile> res = index.find(IndexedFile.class, filter, sorter, sublist, new ArrayList<>());

    for (IndexedFile sf : res.getResults()) {
      System.out.println(sf.toString());
    }
  }

  private static void printEvents(Sorter sorter, Sublist sublist) throws GenericException, RequestNotValidException {
    Filter filter = new Filter(
      new SimpleFilterParameter(RodaConstants.PRESERVATION_EVENT_TYPE, "format identification"));
    IndexResult<IndexedPreservationEvent> res = index.find(IndexedPreservationEvent.class, filter, sorter, sublist,
      new ArrayList<>());

    for (IndexedPreservationEvent ipe : res.getResults()) {
      System.out.println(ipe.toString());
    }
  }

  private static void printAgents(Sorter sorter, Sublist sublist) throws GenericException, RequestNotValidException {
    Filter filter = new Filter(
      new SimpleFilterParameter(RodaConstants.PRESERVATION_AGENT_TYPE, PreservationAgentType.SOFTWARE.toString()));
    IndexResult<IndexedPreservationAgent> res = index.find(IndexedPreservationAgent.class, filter, sorter, sublist,
      new ArrayList<>());

    for (IndexedPreservationAgent ipa : res.getResults()) {
      System.out.println(ipa.toString());
    }
  }

  private static String readPassword(final String message) throws IOException {
    final Console console = System.console();
    if (console == null) {
      final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
      System.out.print(String.format("%s (INSECURE - password will be shown): ", message));
      return reader.readLine();
    } else {
      return new String(console.readPassword("%s: ", message));
    }
  }

  private static void resetAdminAccess() throws GenericException {
    try {
      final String password = readPassword("New admin password");
      final String passwordConfirmation = readPassword("Repeat admin password");
      if (password.equals(passwordConfirmation)) {
        RodaCoreFactory.ldapUtility.resetAdminAccess(password);
        try {
          indexUsersAndGroupsFromLDAP();
        } catch (final Exception e) {
          LOGGER.warn("Error reindexing users and groups - " + e.getMessage(), e);
          System.err.println("Error reindexing users and groups (" + e.getMessage() + ").");
        }
        System.out.println("Password for 'admin' changed successfully.");
      } else {
        throw new GenericException("Passwords don't match.");
      }
    } catch (final IOException e) {
      throw new GenericException(e.getMessage(), e);
    } finally {
      try {
        RodaCoreFactory.shutdown();
      } catch (final Exception e) {
        e.printStackTrace(System.err);
      }
    }
  }

  private static void printMainUsage() {
    System.err.println("WARNING: if using Apache Solr embedded, the index related commands");
    System.err.println("cannot be run while RODA is running (i.e. deployed in Tomcat for example).");
    System.err.println("Stop RODA before running index commands.");
    System.err.println();
    System.err.println("Usage: java -jar roda-core.jar command [arguments]");
    System.err.println("Available commands:");
    System.err.println("\tindex");
    System.err
      .println("\t\treindex aip|job|risk|agent|format|notification|transferred_resources|actionlogs|users_and_groups");
    System.err.println("\t\tlist users|groups|sips|file");
    System.err.println("\torphans [newParentID]");
    System.err.println("\tfixity");
    System.err.println("\tantivirus");
    System.err.println("\tpremisskeleton");
    System.err.println("\treset admin");
    System.err.println("\tmigrate model");
    System.err.println("\tmigrate index");
  }

  private static void printResetUsage() {
    System.err.println("Reset command parameters:");
    System.err.println("\tadmin - resets admin user password and grant it all permissions.");
  }

  private static void printMigrateUsage() {
    System.err.println("Migrate command parameters:");
    System.err.println("\tmodel - performs model related migrations.");
  }

  private static void mainMasterTasks(final List<String> args) throws GenericException, RequestNotValidException {
    if ("index".equals(args.get(0))) {
      if ("list".equals(args.get(1)) && ("users".equals(args.get(2)) || "groups".equals(args.get(2)))) {
        final Filter filter = new Filter(
          new SimpleFilterParameter(RodaConstants.MEMBERS_IS_USER, "users".equals(args.get(2)) ? "true" : "false"));
        printIndexMembers(args, filter, null, new Sublist(0, 10000), null);
      } else if ("list".equals(args.get(1)) && ("sips".equals(args.get(2)))) {
        printCountSips(null, new Sublist(0, 10000), null);
      } else if ("list".equals(args.get(1)) && ("file".equals(args.get(2)))) {
        printFiles(null, new Sublist(0, 10000));
      } else if ("list".equals(args.get(1)) && ("event".equals(args.get(2)))) {
        printEvents(null, new Sublist(0, 10000));
      } else if ("list".equals(args.get(1)) && ("agent".equals(args.get(2)))) {
        printAgents(null, new Sublist(0, 10000));
      } else if ("query".equals(args.get(1)) && args.size() == 4 && StringUtils.isNotBlank(args.get(2))
        && StringUtils.isNotBlank(args.get(3))) {
        runSolrQuery(args);
      } else if ("reindex".equals(args.get(1))) {
        runReindex(args);
      }
    } else if ("reset".equals(args.get(0))) {
      final List<String> resetParams = args.subList(1, args.size());
      if (resetParams.isEmpty()) {
        printResetUsage();
      } else {
        final String resetParam = resetParams.get(0);
        if ("admin".equals(resetParam)) {
          resetAdminAccess();
        } else {
          System.err.println("ERROR: Unknown parameter '" + resetParam + "'");
          printResetUsage();
        }
      }
    } else if ("migrate".equals(args.get(0))) {
      final List<String> migrateParams = args.subList(1, args.size());
      if (migrateParams.isEmpty()) {
        printMigrateUsage();
      } else {
        final String migrateParam = migrateParams.get(0);
        MigrationManager migrationManager = new MigrationManager(RodaCoreFactory.dataPath);
        if ("model".equals(migrateParam)) {
          migrationManager.setupModelMigrations();
          migrationManager.performModelMigrations();
        } else {
          printMigrateUsage();
        }
      }
    } else {
      printMainUsage();
    }
  }

  private static void preInstantiateSteps(List<String> args) {
    if (!args.isEmpty() && "migrate".equals(args.get(0))) {
      migrationMode = true;
    }
  }

  public static void main(final String[] argsArray)
    throws InterruptedException, GenericException, RequestNotValidException {
    final List<String> args = Arrays.asList(argsArray);

    preInstantiateSteps(args);
    instantiate();
    if (getNodeType() == NodeType.MASTER) {
      if (!args.isEmpty()) {
        mainMasterTasks(args);
      } else {
        printMainUsage();
      }
    } else if (getNodeType() == NodeType.WORKER) {
      Thread.currentThread().join();
    } else {
      printMainUsage();
    }

    System.exit(0);
  }
}