package automately.core.file.nio; import automately.core.data.User; import automately.core.file.VirtualFile; import automately.core.file.VirtualFileStore; import com.hazelcast.core.EntryEvent; import com.hazelcast.core.ILock; import com.hazelcast.core.IMap; import com.hazelcast.map.listener.EntryEvictedListener; import com.hazelcast.map.listener.EntryRemovedListener; import com.hazelcast.map.listener.EntryUpdatedListener; import com.hazelcast.query.Predicate; import com.hazelcast.query.Predicates; import io.jsync.app.core.Cluster; import io.jsync.buffer.Buffer; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.FileTime; import java.nio.file.attribute.UserPrincipalLookupService; import java.nio.file.spi.FileSystemProvider; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import static automately.core.file.VirtualFileService.getFileStore; import static automately.core.file.VirtualFileSystem.deleteUserFile; import static automately.core.file.VirtualFileSystem.writeFileData; public class UserFileSystem extends FileSystem implements EntryUpdatedListener<String, VirtualFile>, EntryRemovedListener<String, VirtualFile>, EntryEvictedListener<String, VirtualFile> { final UserFilePath rootFilePath = new UserFilePath(this, "/".getBytes()); private final Cluster cluster; private final IMap<String, VirtualFile> files; private final Map<String, VirtualFile> locallyCachedFiles; private UserFileSystemProvider provider; private User user; UserFileSystem(UserFileSystemProvider provider, User user, Cluster cluster) { super(); this.files = cluster.data().persistentMap("files"); // We need to add an entry listener for caching this.files.addEntryListener(this, Predicates.equal("userToken", user.token()), true); this.locallyCachedFiles = new ConcurrentHashMap<>(); this.provider = provider; this.user = user; this.cluster = cluster; } @Override public FileSystemProvider provider() { return provider; } @Override public void close() throws IOException { } @Override public boolean isOpen() { return true; } @Override public boolean isReadOnly() { return false; } @Override public String getSeparator() { return "/"; } @Override public Iterable<Path> getRootDirectories() { return Collections.singleton(rootFilePath); } @Override public Iterable<FileStore> getFileStores() { throw new UnsupportedOperationException(); } @Override public Set<String> supportedFileAttributeViews() { Set<String> attrs = new LinkedHashSet<>(); //attrs.add("basic"); attrs.add("posix"); return attrs; } @Override public UserFilePath getPath(String first, String... more) { String path; if (more.length == 0) { path = first; } else { StringBuilder sb = new StringBuilder(); sb.append(first); for (String segment : more) { sb.append(segment); } path = sb.toString(); } if (path.endsWith("/.")) { path = path.substring(0, path.length() - 1); } path = path.trim(); if (path.equals("/")) { return rootFilePath; } return new UserFilePath(this, path.getBytes()); } @Override public PathMatcher getPathMatcher(String syntaxAndPattern) { throw new UnsupportedOperationException(); } @Override public UserPrincipalLookupService getUserPrincipalLookupService() { throw new UnsupportedOperationException(); } @Override public WatchService newWatchService() throws IOException { throw new UnsupportedOperationException(); } private void copyOrMove(Path source, Path target, boolean shouldMove, CopyOption... options) throws IOException { Path absoluteSource = source.toAbsolutePath(); Path[] absoluteTarget = new Path[]{target.toAbsolutePath()}; // Are they the same? If so continue as normal if (absoluteSource.compareTo(absoluteTarget[0]) == 0) { return; } // This will make sure we have access to the parent file // If not then it'll throw an error. checkAccess(absoluteTarget[0].getParent()); // Let's get a file reference or throw an error if the source // file does not exist VirtualFile sourceFile = getFile(absoluteSource); boolean shouldReplace = false; for (CopyOption option : options) { if (option instanceof StandardCopyOption) { StandardCopyOption standard = (StandardCopyOption) option; if (standard.compareTo(StandardCopyOption.REPLACE_EXISTING) == 0) { shouldReplace = true; } } } try { // This will throw FileNotFoundException and break into // catch statement if it's not found VirtualFile targetFile = getFile(absoluteTarget[0]); if(shouldMove){ ILock fileLock = cluster.hazelcast().getLock("fs.file.lock." + targetFile.token()); // We cannot delete this file if it is locked. if(fileLock.isLocked()){ throw new IOException(absoluteTarget[0].toString() + " cannot be moved!"); } } // Is the source a directory??? If so we should check if the target file // is a directory. if (sourceFile.isDirectory) { if (targetFile.isDirectory) { try { // TODO fix this?? newDirectoryStream(absoluteSource, null).forEach(path -> { try { copyOrMove(path.toAbsolutePath(), absoluteTarget[0].resolve(path.getFileName()), shouldMove, options); } catch (IOException ignored) { } }); if(shouldMove){ delete(absoluteSource); } } catch (Exception e) { if (cluster.config().isDebug()) { e.printStackTrace(); } throw e; } } else { throw new IOException("Cannot copy directory " + sourceFile + " to " + targetFile + " since it's regular file."); } } else { // This means source file is a regular file if (!targetFile.isDirectory) { // Should we replace the file since it already exists? if(shouldReplace){ try { // This means we are moving the file if(shouldMove){ deleteUserFile(user, targetFile); sourceFile.pathAlias = targetFile.pathAlias; sourceFile.name = targetFile.name; files.set(sourceFile.token(), sourceFile); } else { // This means we are simply updating the file with new data writeFileData(targetFile, getFileStore().readRawData(sourceFile)); } return; } catch (Exception e) { if (cluster.config().isDebug()) { e.printStackTrace(); } throw e; } } // This means the target file already exists and we can't replace it throw new FileAlreadyExistsException(absoluteTarget[0].toString()); } else { // Target file is a directory try { // We can update the absolute target to a direct file absoluteTarget[0] = absoluteTarget[0].resolve(sourceFile.name); // Since the target file is a directory we should check // If a file already exist since we are copying/moving a regular file VirtualFile newTarget = getFile(absoluteTarget[0]); if(newTarget != null){ if(!newTarget.isDirectory){ if(shouldReplace) { if(shouldMove){ // We can simply move the file over deleteUserFile(user, newTarget); sourceFile.pathAlias = newTarget.pathAlias; sourceFile.name = newTarget.name; files.set(sourceFile.token(), sourceFile); } else { writeFileData(newTarget, getFileStore().readRawData(sourceFile)); } return; } else { throw new FileAlreadyExistsException(absoluteTarget[0].toString()); } } else { // We cannot copy the file if the new target // is a directory throw new FileAlreadyExistsException(absoluteTarget[0].toString()); } } VirtualFile newFile = VirtualFile.copy(sourceFile); newFile.pathAlias = getDirStr(absoluteTarget[0]); writeFileData(newFile, getFileStore().readRawData(sourceFile)); } catch (Exception e) { if (cluster.config().isDebug()) { e.printStackTrace(); } throw e; } } } } catch (NoSuchFileException | FileNotFoundException ignored) { // The source file is a directory and the target directory // does not exist so let's go ahead and copy files to it. if (sourceFile.isDirectory) { try { createDirectory(absoluteTarget[0]); newDirectoryStream(absoluteSource, null).forEach(path -> { try { copyOrMove(path.toAbsolutePath(), absoluteTarget[0].resolve(path.getFileName()), shouldMove, options); } catch (IOException e) { if (cluster.config().isDebug()) { e.printStackTrace(); } } }); if(shouldMove){ delete(absoluteSource); } } catch (Exception e) { if (cluster.config().isDebug()) { e.printStackTrace(); } throw e; } } else { // The source file is a file and the target // does not exist try { // This essentially means we are moving the file if(shouldMove){ // We can simply set the new pathAlias and name since we are moving it sourceFile.pathAlias = getDirStr(absoluteTarget[0].getParent()); sourceFile.name = absoluteTarget[0].getFileName().toString(); files.set(sourceFile.token(), sourceFile); } else { // Create a copy of the sourceFile and then write the data since its a copy VirtualFile file = VirtualFile.copy(sourceFile); file.pathAlias = getDirStr(absoluteTarget[0].getParent()); file.name = absoluteTarget[0].getFileName().toString(); writeFileData(file, getFileStore().readRawData(sourceFile)); } } catch (Exception e) { if (cluster.config().isDebug()) { e.printStackTrace(); } throw e; } } } } public void copy(Path source, Path target, CopyOption... options) throws IOException { copyOrMove(source, target, false, options); } public void move(Path source, Path target, CopyOption... options) throws IOException { copyOrMove(source, target, true, options); } public void delete(Path dir) throws IOException { Path absolute = dir.toAbsolutePath(); if (absolute.toString().equals("/")) { throw new IOException(absolute.toString() + " is not an empty directory."); } String filePath = getDirStr(absolute.getParent()); String fileName = absolute.getFileName().toString(); Collection<VirtualFile> vals = files.values(Predicates.or( Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", getDirStr(absolute))), Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", filePath), Predicates.equal("name", fileName)))); if (vals.size() > 1) { throw new IOException(absolute.toString() + " is not an empty directory."); } Iterator<VirtualFile> it = vals.iterator(); if (!it.hasNext()) { throw new NoSuchFileException(absolute.toString()); } VirtualFile file = it.next(); ILock fileLock = cluster.hazelcast().getLock("fs.file.lock." + file.token()); // We cannot delete this file if it is locked. if(fileLock.isLocked()){ throw new IOException(absolute.toString() + " cannot be deleted!"); } deleteUserFile(user, file, true); } public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException { // TODO honor attributes?? Path absolute = dir.toAbsolutePath(); if (absolute.toString().equals("/")) { throw new FileAlreadyExistsException("/"); } checkAccess(absolute.getParent()); String dirPath = getDirStr(absolute.getParent()); String dirName = absolute.getFileName().toString(); Predicate predicate = Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", dirPath), Predicates.equal("name", dirName)); Iterator<VirtualFile> it = files.values(predicate).iterator(); if (it.hasNext()) { throw new FileAlreadyExistsException(absolute.toString()); } VirtualFile file = new VirtualFile(); file.userToken = user.token(); file.pathAlias = dirPath; file.name = dirName; file.isDirectory = true; file.type = "text/directory"; file.size = 0; // Write an empty file because we still need a reference writeFileData(file, new Buffer()); } public DirectoryStream<Path> newDirectoryStream(final Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException { Path absolute = dir.toAbsolutePath(); // Let's make sure we can access the parent checkAccess(absolute.getParent()); Predicate predicate; if (absolute.toString().equals("/")) { predicate = Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", "/")); } else { String dirPath = getDirStr(absolute.getParent()); String dirName = absolute.getFileName().toString(); predicate = Predicates.and(Predicates.equal("userToken", user.token()), Predicates.or(Predicates.equal("pathAlias", getDirStr(absolute)), Predicates.and( Predicates.equal("pathAlias", dirPath), Predicates.equal("name", dirName), Predicates.equal("isDirectory", true)))); } Collection<VirtualFile> vals = files.values(predicate); // If the path is / and empty we can // return an empty directory stream if (vals.size() == 0 && absolute.toString().equals("/")) { return new DirectoryStream<Path>() { @Override public Iterator<Path> iterator() { return new Iterator<Path>() { @Override public boolean hasNext() { return false; } @Override public Path next() { return null; } }; } @Override public void close() throws IOException { // This does nothing } }; } Optional<VirtualFile> findFirst = vals.stream().findFirst(); if (!findFirst.isPresent()) { throw new NoSuchFileException(absolute.toString()); } VirtualFile firstFile = findFirst.get(); if (!firstFile.isDirectory && vals.size() == 1 && !absolute.toString().equals("/")) { throw new IOException(absolute.toString() + " is a regular file."); } if (vals.size() == 1 && !absolute.toString().equals("/")) { return new DirectoryStream<Path>() { @Override public Iterator<Path> iterator() { return new Iterator<Path>() { @Override public boolean hasNext() { return false; } @Override public Path next() { return null; } }; } @Override public void close() throws IOException { // This does nothing } }; } return new DirectoryStream<Path>() { @Override public Iterator<Path> iterator() { return new Iterator<Path>() { private Iterator<VirtualFile> files = vals.iterator(); private Path nextPath; private Path getNextPath() { while (files.hasNext()) { VirtualFile nextFile = files.next(); if (!absolute.toString().equals("/") && nextFile.pathAlias.equals(getDirStr(absolute.getParent())) && nextFile.name.equals(absolute.getFileName().toString())) { continue; } UserFilePath nextPath = new UserFilePath(UserFileSystem.this, nextFile); if (filter != null) { try { if (!filter.accept(nextPath)) { continue; } } catch (IOException ignored) { } } // TODO let's add a cached file reference to ? speed things up locallyCachedFiles.put(nextFile.pathAlias + nextFile.name, nextFile); // Let's go ahead and add this to the cached return nextPath; } return null; } @Override public boolean hasNext() { if (nextPath == null) { Path newNextPath = getNextPath(); if (newNextPath != null) { nextPath = newNextPath; } } return nextPath != null; } @Override public Path next() { if (hasNext()) { Path newNextPath = nextPath; nextPath = null; return newNextPath; } return null; } @Override public void remove() { throw new UnsupportedOperationException(); } }; } @Override public void close() throws IOException { // This does nothing } }; } public <A extends BasicFileAttributes> SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>[] attrs) throws IOException { return newFileChannel(path, options, attrs); } public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>[] attrs) throws IOException { Path absolute = path.toAbsolutePath(); if (absolute.toString().equals("/")) { throw new IOException(absolute.toString() + " is a directory."); } // TODO honor attributes // We need to make sure the parent exists checkAccess(absolute.getParent()); String filePath = getDirStr(absolute.getParent()); String fileName = absolute.getFileName().toString(); VirtualFile file = null; // This means the file exists and it is in fact in the cache if(locallyCachedFiles.containsKey(absolute.toString())){ file = locallyCachedFiles.get(absolute.toString()); } else { Predicate predicate = Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", filePath), Predicates.equal("name", fileName)); Iterator<VirtualFile> it = files.values(predicate).iterator(); // This means the file does not exist... if (!it.hasNext()) { if (options != null) { if ((options.contains(StandardOpenOption.CREATE) || options.contains(StandardOpenOption.CREATE_NEW))) { options.remove(StandardOpenOption.CREATE); options.remove(StandardOpenOption.CREATE_NEW); // Let's make sure the parent exists.. checkAccess(absolute.getParent()); // This means we can go ahead and create the file file = new VirtualFile(); file.userToken = user.token(); file.name = fileName; file.pathAlias = filePath; file.isPublic = false; file.isDirectory = false; // We can create the new file. file = writeFileData(file, new Buffer()); } } } else { file = it.next(); } } if(file == null){ throw new FileNotFoundException(absolute.toString()); } if (file.isDirectory) { throw new IOException(absolute + " is a directory!"); } if (options != null) { if ((options.contains(StandardOpenOption.CREATE_NEW))) { throw new FileAlreadyExistsException(absolute.toString()); } else if ((options.contains(StandardOpenOption.CREATE))) { options.remove(StandardOpenOption.CREATE); // We let"s "create a new file" file = writeFileData(file, new Buffer()); } } try { VirtualFileStore store = getFileStore(); File realFile = store.toFile(file); // This will go ahead and assume the file does not exist. if (realFile == null) { file = writeFileData(file, new Buffer()); realFile = store.toFile(file); } List<OpenOption> openOptions = new LinkedList<>(); openOptions.add(StandardOpenOption.CREATE); openOptions.add(StandardOpenOption.READ); if(options != null && options.contains(StandardOpenOption.WRITE)){ openOptions.add(StandardOpenOption.WRITE); } // This will help support reading and writing files a lot better return FileChannel.open(realFile.toPath(), openOptions.toArray(new OpenOption[openOptions.size()])); } catch (Exception e) { return UserFileChannel.newChannel(file, options != null && options.contains(StandardOpenOption.READ) ? "ro" : "rw"); } } public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException { if (!(path instanceof UserFilePath)) { throw new ProviderMismatchException(); } if(attribute.equals("isPublic")){ UserFilePath userPath = (UserFilePath) path; VirtualFile realFile = getFile(userPath); if(realFile != null){ realFile.isPublic = Boolean.valueOf(value.toString()); locallyCachedFiles.put(userPath.toString(), realFile); files.set(realFile.token(), realFile); } } } public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> clazz, LinkOption... options) throws IOException { if (!(path instanceof UserFilePath)) { throw new ProviderMismatchException(); } return (A) readAttributes((UserFilePath) path); } public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException { if (!(path instanceof UserFilePath)) { throw new ProviderMismatchException(); } UserFilePath absolute = (UserFilePath) path.toAbsolutePath(); if (attributes == null || attributes.isEmpty()) { return new HashMap<>(); // I guess we return an emoty map???? } // TODO attempt to retrieve cached file VirtualFile realFile = getFile(path); Map<String, Object> attrs = new HashMap<>(); if (attributes.startsWith("basic:") || attributes.startsWith("posix:")) { String[] attributeArray = attributes.split(":")[1].split(","); for (String attribute : attributeArray) { if (realFile != null) { if (attribute.equals("size") || attribute.equals("*")) { attrs.put("size", realFile.size); } if (attribute.equals("isOther") || attribute.equals("*")) { attrs.put("isOther", false); } if (attribute.equals("isRegularFile") || attribute.equals("*")) { attrs.put("isRegularFile", !realFile.isDirectory); } if (attribute.equals("isDirectory") || attribute.equals("*")) { attrs.put("isDirectory", realFile.isDirectory); } if (attribute.equals("isSymbolicLink") || attribute.equals("*")) { attrs.put("isSymbolicLink", false); } if (attribute.equals("fileKey") || attribute.equals("*")) { attrs.put("fileKey", realFile.token()); } if (attribute.equals("lastAccessTime") || attribute.equals("*")) { attrs.put("lastAccessTime", FileTime.from(realFile.updated.toInstant())); } if (attribute.equals("lastModifiedTime") || attribute.equals("*")) { attrs.put("lastModifiedTime", FileTime.from(realFile.updated.toInstant())); } if (attribute.equals("creationTime") || attribute.equals("*")) { attrs.put("creationTime", FileTime.from(realFile.created.toInstant())); } } else if (absolute.toString().equals("/")) { if (attribute.equals("size") || attribute.equals("*")) { attrs.put("size", 0); } if (attribute.equals("isOther") || attribute.equals("*")) { attrs.put("isOther", false); } if (attribute.equals("isRegularFile") || attribute.equals("*")) { attrs.put("isRegularFile", false); } if (attribute.equals("isDirectory") || attribute.equals("*")) { attrs.put("isDirectory", true); } if (attribute.equals("isSymbolicLink") || attribute.equals("*")) { attrs.put("isSymbolicLink", false); } if (attribute.equals("fileKey") || attribute.equals("*")) { attrs.put("fileKey", "/"); } if (attribute.equals("lastAccessTime") || attribute.equals("*")) { attrs.put("lastAccessTime", FileTime.from(new Date().toInstant())); } if (attribute.equals("lastModifiedTime") || attribute.equals("*")) { attrs.put("lastModifiedTime", FileTime.from(new Date().toInstant())); } if (attribute.equals("creationTime") || attribute.equals("*")) { attrs.put("creationTime", FileTime.from(new Date().toInstant())); } } if (attributes.startsWith("posix:")) { if (attribute.equals("owner") || attribute.equals("*")) { attrs.put("owner", user.username); } if (attribute.equals("group") || attribute.equals("*")) { attrs.put("group", user.username); } if (attribute.equals("permissions") || attribute.equals("*")) { attrs.put("permissions", UserFileAttributes.DEFAULT_PERMISSIONS); } } } } return attrs; } private UserFileAttributes readAttributes(UserFilePath path) throws IOException { UserFilePath absolute = path.toAbsolutePath(); // This should help speed up readAttributes if (absolute.toString().equals("/")) { return new UserFileAttributes(this, null); } // TODO let's attempt to retrieve a cached version of attributes String filePath = getDirStr(absolute.getParent()); String fileName = absolute.getFileName().toString(); Predicate predicate = Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", filePath), Predicates.equal("name", fileName)); Collection<VirtualFile> vals = files.values(predicate); // Check if the path even exists if (vals.size() == 0) { throw new NoSuchFileException(absolute.toString()); } // We can return the regular file return new UserFileAttributes(this, vals.iterator().next()); } public void checkAccess(Path path, AccessMode... modes) throws IOException { if (path == null) { return; // if it's null then we aren't going to look for it } Path absolute = path.toAbsolutePath(); if (absolute.toString().equals("/")) { return; } // This means the file exists and it is in fact in the cache if(locallyCachedFiles.containsKey(absolute.toString())){ return; } String filePath = getDirStr(absolute.getParent()); String fileName = absolute.getFileName().toString(); Predicate predicate = Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", filePath), Predicates.equal("name", fileName)); Collection<VirtualFile> vals = files.values(predicate); Iterator<VirtualFile> it = vals.iterator(); if (!it.hasNext()) { throw new NoSuchFileException(absolute.toString()); } // This file is now cached... locallyCachedFiles.put(absolute.toString(), it.next()); } private String getDirStr(Path path) { if(path == null){ return "/"; } if (!path.isAbsolute()) { path = path.toAbsolutePath(); } String dirStr = path.toString(); if (!dirStr.endsWith("/")) { dirStr += "/"; } return dirStr; } public VirtualFile getFile(Path path) throws IOException { Path absolute = path.toAbsolutePath(); if (absolute.toString().equals("/")) { return null; // We can go ahead and return null by default } // If the file exists in the cache // Then we can go ahead and assume we can move it if(locallyCachedFiles.containsKey(absolute.toString())){ return locallyCachedFiles.get(absolute.toString()); } String filePath = getDirStr(absolute.getParent()); String fileName = absolute.getFileName().toString(); Predicate predicate = Predicates.and(Predicates.equal("userToken", user.token()), Predicates.equal("pathAlias", filePath), Predicates.equal("name", fileName)); // Can we access this file or directory? Iterator<VirtualFile> it = files.values(predicate).iterator(); if (!it.hasNext()) { throw new FileNotFoundException(absolute.toString()); } VirtualFile file = it.next(); locallyCachedFiles.put(absolute.toString(), file); return file; } public User getUser() { return user; } @Override public void entryUpdated(EntryEvent<String, VirtualFile> event) { // If the entry is a regular file let's attempt to update the parent files updated.. VirtualFile file = event.getValue(); String localKey = file.pathAlias + file.name; // Let's check for the old value in case the path alias or name has changed String oldLocalKey = event.getOldValue() != null ? event.getOldValue().pathAlias + event.getOldValue().name : null; if(locallyCachedFiles.containsKey(localKey) || (oldLocalKey != null && locallyCachedFiles.containsKey(oldLocalKey))){ locallyCachedFiles.put(localKey, file); // We can remove the old key. if(oldLocalKey != null && !localKey.equals(oldLocalKey)){ locallyCachedFiles.remove(oldLocalKey); } } } @Override public void entryRemoved(EntryEvent<String, VirtualFile> event) { // ENSURE GET OLD VALUE VirtualFile file = event.getOldValue(); if(file != null){ String localKey = file.pathAlias + file.name; locallyCachedFiles.remove(localKey); } } @Override public void entryEvicted(EntryEvent<String, VirtualFile> event) { entryRemoved(event); } }