DefaultResourceProvider.java

package eu.europeana.cloud.migrator.provider;

import eu.europeana.cloud.common.model.DataProviderProperties;
import eu.europeana.cloud.migrator.ResourceMigrator;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

public abstract class DefaultResourceProvider
        implements ResourceProvider {

    // Keys for data provider properties
    public static final String ORGANISATION_NAME_KEY = "organisationName";

    public static final String OFFICIAL_ADDRESS_KEY = "officialAddress";

    public static final String ORGANISATION_WEBSITE_KEY = "organisationWebsite";

    public static final String ORGANISATION_WEBSITE_URL_KEY = "organisationWebsiteURL";

    public static final String DIGITAL_LIBRARY_WEBSITE_KEY = "digitalLibraryWebsite";

    public static final String DIGITAL_LIBRARY_WEBSITE_URL_KEY = "digitalLibraryURL";

    public static final String CONTACT_PERSON_KEY = "contactPerson";

    public static final String REMARKS_KEY = "remarks";

    // Default data provider properties
    public static final String DEFAULT_ORGANISATION_NAME = "Example Organisation";

    public static final String DEFAULT_OFFICIAL_ADDRESS = "Example Address";

    public static final String DEFAULT_ORGANISATION_WEBSITE = "Example Website";

    public static final String DEFAULT_ORGANISATION_WEBSITE_URL = "http://www.example.com";

    public static final String DEFAULT_DIGITAL_LIBRARY_WEBSITE = "Example DL Website";

    public static final String DEFAULT_DIGITAL_LIBRARY_WEBSITE_URL = "http://www.example.com/digital";

    public static final String DEFAULT_CONTACT_PERSON = "John Example";

    public static final String DEFAULT_REMARKS = "Example remarks";

    public static final String PROPERTIES_EXTENSION = ".properties";

    public static final int DEFAULT_LIST_SIZE = 500;

    // Logger
    private static final Logger logger = Logger.getLogger(DefaultResourceProvider.class);

    // Representation name from configuration
    protected String representationName;

    // Mapping file storing the association between local identifiers and files/directories in the filesystem
    protected String mappingFile;

    // list of location retrieved from configuration file
    protected List<URI> locations;

    // local indicator = when false the resource location is remote
    protected boolean local;

    // map associating resource provider with its location
    private Map<String, URI> providersLocation;

    // data provider identifier, may be null, if null the identifier is derived from the file path, otherwise it is used for all files
    protected String dataProviderId;

    protected DefaultResourceProvider(String representationName, String mappingFile, String locations, String dataProviderId) {
        if (representationName == null)
            throw new IllegalArgumentException("Representation name cannot be null!");
        this.representationName = representationName;
        this.mappingFile = mappingFile;
        this.locations = new ArrayList<URI>();
        this.providersLocation = new HashMap<String, URI>();
        this.local = detectLocations(locations);
        this.dataProviderId = dataProviderId != null ? (dataProviderId.isEmpty() ? null : dataProviderId) : null;
    }

    private boolean detectLocations(String locations) {
        boolean allLocal = true;
        boolean firstChange = true;
        String[] locs = locations.split(";");
        for (String loc : locs) {
            URI location;
            try {
                location = new URI(loc);
                this.locations.add(location);
            } catch (URISyntaxException e) {
                logger.error("URI " + loc + " is not valid.", e);
                continue;
            }
            String scheme = location.getScheme();
            boolean locationLocal = (scheme == null || scheme.isEmpty() || scheme.toLowerCase().equals("file"));
            if (locationLocal != allLocal) {
                if (firstChange) {
                    firstChange = false;
                    allLocal = locationLocal;
                } else {
                    logger.error("All locations must be either local or remote");
                    throw new IllegalArgumentException("All locations must be either local or remote. locations: " + locations);
                }
            }
        }
        return allLocal;
    }

    protected DataProviderProperties getDataProviderPropertiesFromFile(File dpFile) {
        Properties props = loadPropertiesFile(dpFile);
        return new DataProviderProperties(props.getProperty(ORGANISATION_NAME_KEY, DEFAULT_ORGANISATION_NAME),
                props.getProperty(OFFICIAL_ADDRESS_KEY, DEFAULT_OFFICIAL_ADDRESS),
                props.getProperty(ORGANISATION_WEBSITE_KEY, DEFAULT_ORGANISATION_WEBSITE),
                props.getProperty(ORGANISATION_WEBSITE_URL_KEY, DEFAULT_ORGANISATION_WEBSITE_URL),
                props.getProperty(DIGITAL_LIBRARY_WEBSITE_KEY, DEFAULT_DIGITAL_LIBRARY_WEBSITE),
                props.getProperty(DIGITAL_LIBRARY_WEBSITE_URL_KEY, DEFAULT_DIGITAL_LIBRARY_WEBSITE_URL),
                props.getProperty(CONTACT_PERSON_KEY, DEFAULT_CONTACT_PERSON),
                props.getProperty(REMARKS_KEY, DEFAULT_REMARKS));
    }

    private Properties loadPropertiesFile(File dpFile) {
        Properties props = new Properties();
        InputStream is = null;
        try {
            is = new FileInputStream(dpFile);
            props.load(is);
        } catch (IOException e) {
            logger.error("Problem with file " + dpFile.getAbsolutePath(), e);
        } finally {
            try {
                is.close();
            } catch (IOException e) {
                logger.error("Could not close input stream.", e);
            }
        }
        return props;
    }

    protected DataProviderProperties getDefaultDataProviderProperties() {
        return new DataProviderProperties(DEFAULT_ORGANISATION_NAME,
                DEFAULT_OFFICIAL_ADDRESS,
                DEFAULT_ORGANISATION_WEBSITE,
                DEFAULT_ORGANISATION_WEBSITE_URL,
                DEFAULT_DIGITAL_LIBRARY_WEBSITE,
                DEFAULT_DIGITAL_LIBRARY_WEBSITE_URL,
                DEFAULT_CONTACT_PERSON,
                DEFAULT_REMARKS);
    }

    @Override
    public String getRepresentationName() {
        return representationName;
    }


    @Override
    public List<URI> getLocations() {
        return locations;
    }


    @Override
    public boolean isLocal() {
        return local;
    }

    @Override
    public Map<String, List<FilePaths>> scan() {
        Map<String, List<FilePaths>> paths = new HashMap<String, List<FilePaths>>();
        if (!local) {
            logger.warn("Location is not local. Scanning is not possible.");
            return paths;
        }

        for (URI location : locations) {
            collectPaths(Paths.get(location), paths, location);
            sortPaths(paths);
        }
        return paths;
    }

    private void sortPaths(Map<String, List<FilePaths>> paths) {
        for (List<FilePaths> list : paths.values()) {
            for (FilePaths fp : list)
                fp.sort();
        }
    }

    private void collectPaths(Path rootPath, Map<String, List<FilePaths>> paths, URI location) {
        if (rootPath == null)
            return;

        DirectoryStream<Path> dirStream = null;

        try {
            dirStream = Files.newDirectoryStream(rootPath);

            for (Iterator<Path> i = dirStream.iterator(); i.hasNext(); ) {
                Path path = i.next();
                if (Files.isDirectory(path))
                    collectPaths(path, paths, location);
                else {
                    String absolute = path.toAbsolutePath().toString().replace(ResourceMigrator.WINDOWS_SEPARATOR, ResourceMigrator.LINUX_SEPARATOR);
                    String providerId = getResourceProviderId(absolute);
                    if (absolute.endsWith(providerId + PROPERTIES_EXTENSION))
                        continue;
                    FilePaths providerPaths = getProviderPaths(Paths.get(location).toAbsolutePath().toString().replace(ResourceMigrator.WINDOWS_SEPARATOR, ResourceMigrator.LINUX_SEPARATOR), providerId, paths);
                    //providerPaths.getFullPaths().add(absolute);
                    providerPaths.addPath(absolute);
                    if (providersLocation.get(providerId) == null)
                        providersLocation.put(providerId, location);
                }
            }
        } catch (IOException e) {
            logger.error("There was a problem with opening direcory " + rootPath.toString(), e);
        } finally {
            if (dirStream != null)
                try {
                    dirStream.close();
                } catch (IOException e) {
                    logger.error("There was a problem with closing direcory stream for " + rootPath.toString(), e);
                }
        }
    }

    protected FilePaths getProviderPaths(String location, String providerId, Map<String, List<FilePaths>> paths) {
        if (paths.get(providerId) == null)
            paths.put(providerId, new ArrayList<FilePaths>());
        for (FilePaths p : paths.get(providerId)) {
            if (p.getLocation().equals(location))
                return p;
        }
        FilePaths fp = new FilePaths(location, providerId);
        if (usePathsFile())
            fp.useFile(location.replace(ResourceMigrator.LINUX_SEPARATOR, "_").replace(":", "_") + "_" + providerId);
        paths.get(providerId).add(fp);
        return fp;
    }

    public URI getProvidersLocation(String providerId) {
        return providersLocation.get(providerId);
    }

    /**
     * Default implementation of determining filename from path.
     * The result is either path without location, part of the specified path after "/" or "\" character or path itself.
     *
     * @param location location where path is placed, usually path starts with location
     * @param path     path to file either local or remote in URI syntax
     * @return filename to be stored in ECloud
     */
    @Override
    public String getFilename(String location, String path) {
        if (logger.isDebugEnabled()) {
            logger.debug("Get filename for location: " + location + ", path: " + path);
        }
        int pos = -1;
        if (path.startsWith(location)) {
            path = path.substring(location.length());
            if (logger.isDebugEnabled()) {
                logger.debug("Path without location: " + path);
            }
            pos = path.indexOf(ResourceMigrator.LINUX_SEPARATOR);
            if (pos == -1)
                pos = path.indexOf(ResourceMigrator.WINDOWS_SEPARATOR);
        } else {
            pos = path.lastIndexOf(ResourceMigrator.LINUX_SEPARATOR);
            if (pos == -1)
                pos = path.lastIndexOf(ResourceMigrator.WINDOWS_SEPARATOR);
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Returning: " + path.substring(pos + 1));
        }
        // when pos == -1 whole path is returned, otherwise only part after pos
        return path.substring(pos + 1);
    }

    @Override
    public int getFileCount(String localId) {
        // impossible to determine in anstract class
        return -1;
    }


    /**
     * Default resource provider splits the paths list to several lists of more or less equal size around 500.
     *
     * @param paths
     * @return
     */
    @Override
    public List<FilePaths> split(List<FilePaths> paths) {
        List<FilePaths> result = new ArrayList<FilePaths>();
        for (FilePaths fp : paths) {
            result.addAll(split(fp));
        }
        return result;
    }

    private List<FilePaths> split(FilePaths fp) {
        // size of the list
        int size = fp.size();
        // number of full DEFAULT_LIST_SIZE parts
        int count = size / DEFAULT_LIST_SIZE;
        // size of the last part
        int rest = size % DEFAULT_LIST_SIZE;

        List<FilePaths> result = new ArrayList<FilePaths>();

        // when no need to split return the same file paths object
        if (size <= DEFAULT_LIST_SIZE)
            result.add(fp);
        else {
            for (int i = 0; i < count; i++) {
                FilePaths filePath = new FilePaths(fp.getLocation(), fp.getDataProvider());
                filePath.getFullPaths().addAll(fp.getFullPaths().subList(i * DEFAULT_LIST_SIZE, (i + 1) * DEFAULT_LIST_SIZE < size ? (i + 1) * DEFAULT_LIST_SIZE : size));
                filePath.setIdentifier(String.valueOf(i * DEFAULT_LIST_SIZE) + "_" + String.valueOf((i + 1) * DEFAULT_LIST_SIZE < size ? (i + 1) * DEFAULT_LIST_SIZE : size));
                result.add(filePath);
            }
            if (rest > 0) {
                FilePaths filePath = new FilePaths(fp.getLocation(), fp.getDataProvider());
                filePath.getFullPaths().addAll(fp.getFullPaths().subList(count * DEFAULT_LIST_SIZE, size));
                filePath.setIdentifier(String.valueOf(count * DEFAULT_LIST_SIZE) + "_" + String.valueOf(size));
                result.add(filePath);
            }
        }
        return result;
    }


    @Override
    public boolean usePathsFile() {
        return false;
    }


    @Override
    public Map<String, String> getReversedMapping() {
        return new HashMap<String, String>();
    }
}