/*
* Syncany, www.syncany.org
* Copyright (C) 2011-2015 Philipp C. Heckel <philipp.heckel@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.syncany.plugins.transfer.features;
import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.syncany.config.Config;
import org.syncany.plugins.transfer.FileType;
import org.syncany.plugins.transfer.StorageException;
import org.syncany.plugins.transfer.StorageTestResult;
import org.syncany.plugins.transfer.TransferManager;
import org.syncany.plugins.transfer.TransferPlugin;
import org.syncany.plugins.transfer.files.MultichunkRemoteFile;
import org.syncany.plugins.transfer.files.RemoteFile;
import org.syncany.plugins.transfer.files.RemoteFileAttributes;
import org.syncany.util.ReflectionUtil;
import org.syncany.util.StringUtil;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.hash.Hashing;
/**
* The path aware transfer manager can be used to extend a backend storage
* with the ability to add subfolders to the folders with many files (e.g. multichunks
* or temporary files). This is especially critical if the backend storage has a limit
* on how many files can be stored in a single folder (e.g. the Dropbox plugin).
*
* <p>To enable subfoldering in {@link TransferPlugin}s, the plugin's {@link TransferManager}
* has to be annotated with the {@link PathAware} annotation, and a {@link PathAwareFeatureExtension}
* has to be provided.
*
* <p>The sub-path for a {@link RemoteFile} can then be accessed via the
* {@link PathAwareRemoteFileAttributes} using the {@link RemoteFile#getAttributes(Class)} method.
*
* @see PathAware
* @see PathAwareFeatureExtension
* @see PathAwareRemoteFileAttributes
* @author Christian Roth <christian.roth@port17.de>
*/
public class PathAwareFeatureTransferManager implements FeatureTransferManager {
private static final Logger logger = Logger.getLogger(PathAwareFeatureTransferManager.class.getSimpleName());
private final TransferManager underlyingTransferManager;
private final int subfolderDepth;
private final int bytesPerFolder;
private final char folderSeparator;
private final List<Class<? extends RemoteFile>> affectedFiles;
private final PathAwareFeatureExtension pathAwareFeatureExtension;
public PathAwareFeatureTransferManager(TransferManager originalTransferManager, TransferManager underlyingTransferManager, Config config, PathAware pathAwareAnnotation) {
this.underlyingTransferManager = underlyingTransferManager;
this.subfolderDepth = pathAwareAnnotation.subfolderDepth();
this.bytesPerFolder = pathAwareAnnotation.bytesPerFolder();
this.folderSeparator = pathAwareAnnotation.folderSeparator();
this.affectedFiles = ImmutableList.copyOf(pathAwareAnnotation.affected());
this.pathAwareFeatureExtension = getPathAwareFeatureExtension(originalTransferManager, pathAwareAnnotation);
}
@SuppressWarnings("unchecked")
private PathAwareFeatureExtension getPathAwareFeatureExtension(TransferManager originalTransferManager, PathAware pathAwareAnnotation) {
Class<? extends TransferManager> originalTransferManagerClass = originalTransferManager.getClass();
Class<PathAwareFeatureExtension> pathAwareFeatureExtensionClass = (Class<PathAwareFeatureExtension>) pathAwareAnnotation.extension();
try {
Constructor<?> constructor = ReflectionUtil.getMatchingConstructorForClass(pathAwareFeatureExtensionClass, originalTransferManagerClass);
if (constructor != null) {
return (PathAwareFeatureExtension) constructor.newInstance(originalTransferManager);
}
return pathAwareFeatureExtensionClass.newInstance();
}
catch (InvocationTargetException | InstantiationException | IllegalAccessException | NullPointerException e) {
throw new RuntimeException("Cannot instantiate PathAwareFeatureExtension (perhaps " + pathAwareFeatureExtensionClass + " does not exist?)", e);
}
}
@Override
public void connect() throws StorageException {
underlyingTransferManager.connect();
}
@Override
public void disconnect() throws StorageException {
underlyingTransferManager.disconnect();
}
@Override
public void init(final boolean createIfRequired) throws StorageException {
underlyingTransferManager.init(createIfRequired);
}
@Override
public void download(final RemoteFile remoteFile, final File localFile) throws StorageException {
underlyingTransferManager.download(createPathAwareRemoteFile(remoteFile), localFile);
}
@Override
public void move(final RemoteFile sourceFile, final RemoteFile targetFile) throws StorageException {
final RemoteFile pathAwareSourceFile = createPathAwareRemoteFile(sourceFile);
final RemoteFile pathAwareTargetFile = createPathAwareRemoteFile(targetFile);
if (!createFolder(pathAwareTargetFile)) {
throw new StorageException("Unable to create path for " + pathAwareTargetFile);
}
underlyingTransferManager.move(pathAwareSourceFile, pathAwareTargetFile);
removeFolder(pathAwareSourceFile);
}
@Override
public void upload(final File localFile, final RemoteFile remoteFile) throws StorageException {
final RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile);
if (!createFolder(pathAwareRemoteFile)) {
throw new StorageException("Unable to create path for " + pathAwareRemoteFile);
}
underlyingTransferManager.upload(localFile, pathAwareRemoteFile);
}
@Override
public boolean delete(final RemoteFile remoteFile) throws StorageException {
RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile);
boolean fileDeleted = underlyingTransferManager.delete(pathAwareRemoteFile);
boolean folderDeleted = removeFolder(pathAwareRemoteFile);
return fileDeleted && folderDeleted;
}
@Override
public <T extends RemoteFile> Map<String, T> list(final Class<T> remoteFileClass) throws StorageException {
Map<String, T> filesInFolder = Maps.newHashMap();
String remoteFilePath = getRemoteFilePath(remoteFileClass);
list(remoteFilePath, filesInFolder, remoteFileClass);
return filesInFolder;
}
private <T extends RemoteFile> void list(String remoteFilePath, Map<String, T> remoteFiles, Class<T> remoteFileClass) throws StorageException {
logger.log(Level.INFO, "Listing folder for files matching " + remoteFileClass.getSimpleName() + ": " + remoteFilePath);
Map<String, FileType> folderList = pathAwareFeatureExtension.listFolder(remoteFilePath);
for (Map.Entry<String, FileType> folderListEntry : folderList.entrySet()) {
String fileName = folderListEntry.getKey();
FileType fileType = folderListEntry.getValue();
if (fileType == FileType.FILE) {
try {
remoteFiles.put(fileName, RemoteFile.createRemoteFile(fileName, remoteFileClass));
logger.log(Level.INFO, "- File: " + fileName);
}
catch (StorageException e) {
// We don't care and ignore non-matching files!
}
}
else if (fileType == FileType.FOLDER) {
logger.log(Level.INFO, "- Folder: " + fileName);
String newRemoteFilePath = remoteFilePath + folderSeparator + fileName;
list(newRemoteFilePath, remoteFiles, remoteFileClass);
}
}
}
@Override
public String getRemoteFilePath(Class<? extends RemoteFile> remoteFileClass) {
return underlyingTransferManager.getRemoteFilePath(remoteFileClass);
}
@Override
public StorageTestResult test(boolean testCreateTarget) {
return underlyingTransferManager.test(testCreateTarget);
}
@Override
public boolean testTargetExists() throws StorageException {
return underlyingTransferManager.testTargetExists();
}
@Override
public boolean testTargetCanWrite() throws StorageException {
return underlyingTransferManager.testTargetCanWrite();
}
@Override
public boolean testTargetCanCreate() throws StorageException {
return underlyingTransferManager.testTargetCanCreate();
}
@Override
public boolean testRepoFileExists() throws StorageException {
return underlyingTransferManager.testRepoFileExists();
}
private boolean isFolderizable(Class<? extends RemoteFile> remoteFileClass) {
return affectedFiles.contains(remoteFileClass);
}
private RemoteFile createPathAwareRemoteFile(RemoteFile remoteFile) throws StorageException {
PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = new PathAwareRemoteFileAttributes();
remoteFile.setAttributes(pathAwareRemoteFileAttributes);
if (isFolderizable(remoteFile.getClass())) {
// If remote file is folderizable, i.e. an 'affected file',
// get the sub-path for it
String subPathId = getSubPathId(remoteFile);
String subPath = getSubPath(subPathId);
pathAwareRemoteFileAttributes.setPath(subPath);
}
return remoteFile;
}
/**
* Returns the subpath identifier for this file. For {@link MultichunkRemoteFile}s, this is the
* hex string of the multichunk identifier. For all other files, this is the 128-bit murmur3 hash
* of the full filename (fast algorithm!).
*/
private String getSubPathId(RemoteFile remoteFile) {
if (remoteFile.getClass() == MultichunkRemoteFile.class) {
return StringUtil.toHex(((MultichunkRemoteFile) remoteFile).getMultiChunkId());
}
else {
return StringUtil.toHex(Hashing.murmur3_128().hashString(remoteFile.getName(), Charsets.UTF_8).asBytes());
}
}
private String getSubPath(String fileId) {
StringBuilder path = new StringBuilder();
for (int i = 0; i < subfolderDepth; i++) {
String subPathPart = fileId.substring(i * bytesPerFolder * 2, (i + 1) * bytesPerFolder * 2);
path.append(subPathPart);
path.append(folderSeparator);
}
return path.toString();
}
private String pathToString(Path path) {
return path.toString().replaceAll(File.separator, String.valueOf(folderSeparator));
}
private boolean createFolder(RemoteFile remoteFile) throws StorageException {
PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class);
boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath();
if (notAPathAwareRemoteFile) {
return true;
}
else {
String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath()));
logger.log(Level.INFO, "Remote file is path aware, creating folder " + remoteFilePath);
boolean success = pathAwareFeatureExtension.createPath(remoteFilePath);
return success;
}
}
private boolean removeFolder(RemoteFile remoteFile) throws StorageException {
PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class);
boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath();
if (notAPathAwareRemoteFile) {
return true;
}
else {
String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath()));
logger.log(Level.INFO, "Remote file is path aware, cleaning empty folders at " + remoteFilePath);
boolean success = removeFolder(remoteFilePath);
return success;
}
}
private boolean removeFolder(String folder) throws StorageException {
for(int i = 0; i < subfolderDepth; i++) {
logger.log(Level.FINE, "Removing folder " + folder);
if (pathAwareFeatureExtension.listFolder(folder).size() != 0) {
return true;
}
if (!pathAwareFeatureExtension.removeFolder(folder)) {
return false;
}
folder = folder.substring(0, folder.lastIndexOf(folderSeparator));
}
return true;
}
public static class PathAwareRemoteFileAttributes extends RemoteFileAttributes {
private String path;
public boolean hasPath() {
return path != null;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
}
}