PathAwareFeatureTransferManager.java

/*
 * Syncany, www.syncany.org
 * Copyright (C) 2011-2016 Philipp C. Heckel <philipp.heckel@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.syncany.plugins.transfer.features;

import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.syncany.config.Config;
import org.syncany.plugins.transfer.FileType;
import org.syncany.plugins.transfer.StorageException;
import org.syncany.plugins.transfer.StorageTestResult;
import org.syncany.plugins.transfer.TransferManager;
import org.syncany.plugins.transfer.TransferPlugin;
import org.syncany.plugins.transfer.files.MultichunkRemoteFile;
import org.syncany.plugins.transfer.files.RemoteFile;
import org.syncany.plugins.transfer.files.RemoteFileAttributes;
import org.syncany.util.ReflectionUtil;
import org.syncany.util.StringUtil;

import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.hash.Hashing;

/**
 * The path aware transfer manager can be used to extend a backend storage
 * with the ability to add subfolders to the folders with many files (e.g. multichunks
 * or temporary files). This is especially critical if the backend storage has a limit
 * on how many files can be stored in a single folder (e.g. the Dropbox plugin). 
 * 
 * <p>To enable subfoldering in {@link TransferPlugin}s, the plugin's {@link TransferManager}
 * has to be annotated with the {@link PathAware} annotation, and a {@link PathAwareFeatureExtension}
 * has to be provided.
 * 
 * <p>The sub-path for a {@link RemoteFile} can then be accessed via the
 * {@link PathAwareRemoteFileAttributes} using the {@link RemoteFile#getAttributes(Class)} method.
 * 
 * @see PathAware
 * @see PathAwareFeatureExtension
 * @see PathAwareRemoteFileAttributes
 * @author Christian Roth <christian.roth@port17.de>
 */
public class PathAwareFeatureTransferManager implements FeatureTransferManager {
	private static final Logger logger = Logger.getLogger(PathAwareFeatureTransferManager.class.getSimpleName());

	private final TransferManager underlyingTransferManager;
	
	private final int subfolderDepth;
	private final int bytesPerFolder;
	private final char folderSeparator;
	private final List<Class<? extends RemoteFile>> affectedFiles;
	private final PathAwareFeatureExtension pathAwareFeatureExtension;

	public PathAwareFeatureTransferManager(TransferManager originalTransferManager, TransferManager underlyingTransferManager, Config config, PathAware pathAwareAnnotation) {
		this.underlyingTransferManager = underlyingTransferManager;

		this.subfolderDepth = pathAwareAnnotation.subfolderDepth();
		this.bytesPerFolder = pathAwareAnnotation.bytesPerFolder();
		this.folderSeparator = pathAwareAnnotation.folderSeparator();
		this.affectedFiles = ImmutableList.copyOf(pathAwareAnnotation.affected());

		this.pathAwareFeatureExtension = getPathAwareFeatureExtension(originalTransferManager, pathAwareAnnotation);
	}

	@SuppressWarnings("unchecked")
	private PathAwareFeatureExtension getPathAwareFeatureExtension(TransferManager originalTransferManager, PathAware pathAwareAnnotation) {
		Class<? extends TransferManager> originalTransferManagerClass = originalTransferManager.getClass();
		Class<PathAwareFeatureExtension> pathAwareFeatureExtensionClass = (Class<PathAwareFeatureExtension>) pathAwareAnnotation.extension();

		try {
			Constructor<?> constructor = ReflectionUtil.getMatchingConstructorForClass(pathAwareFeatureExtensionClass, originalTransferManagerClass);

			if (constructor != null) {
				return (PathAwareFeatureExtension) constructor.newInstance(originalTransferManager);
			}

			return pathAwareFeatureExtensionClass.newInstance();
		}
		catch (InvocationTargetException | InstantiationException | IllegalAccessException | NullPointerException e) {
			throw new RuntimeException("Cannot instantiate PathAwareFeatureExtension (perhaps " + pathAwareFeatureExtensionClass + " does not exist?)", e);
		}
	}

	@Override
	public void connect() throws StorageException {
		underlyingTransferManager.connect();
	}

	@Override
	public void disconnect() throws StorageException {
		underlyingTransferManager.disconnect();
	}

	@Override
	public void init(final boolean createIfRequired) throws StorageException {
		underlyingTransferManager.init(createIfRequired);
	}

	@Override
	public void download(final RemoteFile remoteFile, final File localFile) throws StorageException {
		underlyingTransferManager.download(createPathAwareRemoteFile(remoteFile), localFile);
	}

	@Override
	public void move(final RemoteFile sourceFile, final RemoteFile targetFile) throws StorageException {
		final RemoteFile pathAwareSourceFile = createPathAwareRemoteFile(sourceFile);
		final RemoteFile pathAwareTargetFile = createPathAwareRemoteFile(targetFile);

		if (!createFolder(pathAwareTargetFile)) {
			throw new StorageException("Unable to create path for " + pathAwareTargetFile);
		}

		underlyingTransferManager.move(pathAwareSourceFile, pathAwareTargetFile);
		removeFolder(pathAwareSourceFile);
	}

	@Override
	public void upload(final File localFile, final RemoteFile remoteFile) throws StorageException {
		final RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile);

		if (!createFolder(pathAwareRemoteFile)) {
			throw new StorageException("Unable to create path for " + pathAwareRemoteFile);
		}

		underlyingTransferManager.upload(localFile, pathAwareRemoteFile);
	}

	@Override
	public boolean delete(final RemoteFile remoteFile) throws StorageException {
		RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile);
		
		boolean fileDeleted = underlyingTransferManager.delete(pathAwareRemoteFile);
		boolean folderDeleted = removeFolder(pathAwareRemoteFile);

		return fileDeleted && folderDeleted;
	}

	@Override
	public <T extends RemoteFile> Map<String, T> list(final Class<T> remoteFileClass) throws StorageException {
		Map<String, T> filesInFolder = Maps.newHashMap();
		String remoteFilePath = getRemoteFilePath(remoteFileClass);

		list(remoteFilePath, filesInFolder, remoteFileClass);

		return filesInFolder;
	}

	private <T extends RemoteFile> void list(String remoteFilePath, Map<String, T> remoteFiles, Class<T> remoteFileClass) throws StorageException {
		logger.log(Level.INFO, "Listing folder for files matching " + remoteFileClass.getSimpleName() + ": " + remoteFilePath);
		Map<String, FileType> folderList = pathAwareFeatureExtension.listFolder(remoteFilePath);
		
		for (Map.Entry<String, FileType> folderListEntry : folderList.entrySet()) {
			String fileName = folderListEntry.getKey();
			FileType fileType = folderListEntry.getValue();
			
			if (fileType == FileType.FILE) {
				try {
					remoteFiles.put(fileName, RemoteFile.createRemoteFile(fileName, remoteFileClass));
					logger.log(Level.INFO, "- File: " + fileName);					
				}
				catch (StorageException e) {
					// We don't care and ignore non-matching files!
				}
			}
			else if (fileType == FileType.FOLDER) {
				logger.log(Level.INFO, "- Folder: " + fileName);

				String newRemoteFilePath = remoteFilePath + folderSeparator + fileName;
				list(newRemoteFilePath, remoteFiles, remoteFileClass);
			}
		}
	}

	@Override
	public String getRemoteFilePath(Class<? extends RemoteFile> remoteFileClass) {
		return underlyingTransferManager.getRemoteFilePath(remoteFileClass);
	}

	@Override
	public StorageTestResult test(boolean testCreateTarget) {
		return underlyingTransferManager.test(testCreateTarget);
	}

	@Override
	public boolean testTargetExists() throws StorageException {
		return underlyingTransferManager.testTargetExists();
	}

	@Override
	public boolean testTargetCanWrite() throws StorageException {
		return underlyingTransferManager.testTargetCanWrite();
	}

	@Override
	public boolean testTargetCanCreate() throws StorageException {
		return underlyingTransferManager.testTargetCanCreate();
	}

	@Override
	public boolean testRepoFileExists() throws StorageException {
		return underlyingTransferManager.testRepoFileExists();
	}

	private boolean isFolderizable(Class<? extends RemoteFile> remoteFileClass) {
		return affectedFiles.contains(remoteFileClass);
	}

	private RemoteFile createPathAwareRemoteFile(RemoteFile remoteFile) throws StorageException {		
		PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = new PathAwareRemoteFileAttributes();
		remoteFile.setAttributes(pathAwareRemoteFileAttributes);

		if (isFolderizable(remoteFile.getClass())) {
			// If remote file is folderizable, i.e. an 'affected file', 
			// get the sub-path for it
			
			String subPathId = getSubPathId(remoteFile);	
			String subPath = getSubPath(subPathId);
			
			pathAwareRemoteFileAttributes.setPath(subPath);
		}
		
		return remoteFile;
	}

	/**
	 * Returns the subpath identifier for this file. For {@link MultichunkRemoteFile}s, this is the
	 * hex string of the multichunk identifier. For all other files, this is the 128-bit murmur3 hash
	 * of the full filename (fast algorithm!).
	 */
	private String getSubPathId(RemoteFile remoteFile) {
		if (remoteFile.getClass() == MultichunkRemoteFile.class) {
			return StringUtil.toHex(((MultichunkRemoteFile) remoteFile).getMultiChunkId());
		}
		else {
			return StringUtil.toHex(Hashing.murmur3_128().hashString(remoteFile.getName(), Charsets.UTF_8).asBytes());
		}
	}

	private String getSubPath(String fileId) {
		StringBuilder path = new StringBuilder();

		for (int i = 0; i < subfolderDepth; i++) {
			String subPathPart = fileId.substring(i * bytesPerFolder * 2, (i + 1) * bytesPerFolder * 2); 
			
			path.append(subPathPart);
			path.append(folderSeparator);
		}
		
		return path.toString();
	}

	private String pathToString(Path path) {
		return path.toString().replaceAll(File.separator, String.valueOf(folderSeparator));
	}

	private boolean createFolder(RemoteFile remoteFile) throws StorageException {
		PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class);		
		boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath();
		
		if (notAPathAwareRemoteFile) {
			return true;
		} 
		else {
			String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath()));
	
			logger.log(Level.INFO, "Remote file is path aware, creating folder " + remoteFilePath);
			boolean success = pathAwareFeatureExtension.createPath(remoteFilePath);
			
			return success;
		}
	}

	private boolean removeFolder(RemoteFile remoteFile) throws StorageException {
		PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class);
		boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath();
		
		if (notAPathAwareRemoteFile) {
			return true;
		}
		else {
			String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath()));

			logger.log(Level.INFO, "Remote file is path aware, cleaning empty folders at " + remoteFilePath);
			boolean success = removeFolder(remoteFilePath);
		
			return success;
		}
	}

	private boolean removeFolder(String folder) throws StorageException {
		for(int i = 0; i < subfolderDepth; i++) {
			logger.log(Level.FINE, "Removing folder " + folder);

			if (pathAwareFeatureExtension.listFolder(folder).size() != 0) {
				return true;
			}

			if (!pathAwareFeatureExtension.removeFolder(folder)) {
				return false;
			}

			folder = folder.substring(0, folder.lastIndexOf(folderSeparator));
		}

		return true;
	}

	public static class PathAwareRemoteFileAttributes extends RemoteFileAttributes {
		private String path;

		public boolean hasPath() {
			return path != null;
		}

		public String getPath() {
			return path;
		}

		public void setPath(String path) {
			this.path = path;
		}
	}

}