/*
* Copyright (c) 2013, the authors.
*
* This file is part of 'DXFS'.
*
* DXFS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* DXFS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with DXFS. If not, see <http://www.gnu.org/licenses/>.
*/
package nextflow.fs.dx;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.charset.Charset;
import java.nio.file.AccessMode;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryStream;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileStore;
import java.nio.file.FileSystem;
import java.nio.file.FileSystemAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.ProviderMismatchException;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.FileAttributeView;
import java.nio.file.spi.FileSystemProvider;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.fasterxml.jackson.databind.JsonNode;
import nextflow.fs.dx.api.DxApi;
import nextflow.fs.dx.api.DxHttpClient;
import nextflow.fs.dx.api.DxJson;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.params.CoreProtocolPNames;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* NIO2 File system provider for DnaNexus cloud storage
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
* @author Beatriz San Juan <bmsanjuan@gmail.com>
*
*/
public class DxFileSystemProvider extends FileSystemProvider {
private static final Logger log = LoggerFactory.getLogger(DxFileSystemProvider.class);
/**
* The *scheme* defined by this provider
*/
public static String SCHEME = "dxfs";
/**
* Pattern matching a DnaNexus URI e.g. dxfs://container:some/path
*/
public static Pattern URI_PATTERN = Pattern.compile("^" + SCHEME + "://(([^/]+):)?(.*)$");
/**
* Pattern matching a DnaNexus project or context ID
*/
static Pattern CONTEXT_ID_PATTERN = Pattern.compile("^(project-|container-)[a-zA-Z0-9]{24}$");
/**
* Pattern matching a DnaNexus file ID
*/
public static Pattern FILE_ID_PATTERN = Pattern.compile("file-[a-zA-Z0-9]{24}");
/**
* Hold DnaNexus context ID, either a project id or a data container container
* <p>
* Read more
* https://wiki.dnanexus.com/API-Specification-v1.0.0/Data-Containers#
*/
final String defaultContextId;
final ConcurrentHashMap<String,DxFileSystem> fileSystems = new ConcurrentHashMap<>();
final DxApi api;
public DxFileSystemProvider() {
this.defaultContextId = getDefaultDxContextId();
this.api = DxApi.getInstance();
}
protected DxFileSystemProvider( String containerId, DxApi api ) {
this.defaultContextId = containerId;
this.api = api;
}
/**
* Find out the default DnaNexus project id in the default user configuration file,
* i.e. the file {@code $HOME/.dnanexus_config/environment.json}
*
* @return The string value
*/
static String getDefaultDxContextId() {
String result;
@SuppressWarnings({ "unchecked", "rawtypes" })
Map<String, String> props = new HashMap(System.getProperties());
result = getContextIdByMap( props, null );
if( result == null ) {
result = getContextIdByMap( System.getenv(), null );
}
if( result == null ) {
String home = System.getProperty("user.home");
File config = new File(home, ".dnanexus_config/environment.json");
if( !config.exists() ) {
return null;
}
result = getContextIdByConfig(config);
log.debug("Using DX_PROJECT_CONTEXT_ID = {} in config file: {}", result, config);
}
return result;
}
static String getContextIdByMap( Map<String,?> map, String defValue ) {
if( map.containsKey("DX_WORKSPACE_ID") ) {
String result = map.get("DX_WORKSPACE_ID").toString();
log.debug("Using DX_WORKSPACE_ID = {}", result);
return result;
}
else if( map.containsKey("DX_PROJECT_CONTEXT_ID") ) {
String result = map.get("DX_PROJECT_CONTEXT_ID").toString();
log.debug("Using DX_PROJECT_CONTEXT_ID = {}", result);
return result;
}
if( defValue != null ) {
log.debug("Using default context id = {}", defValue);
}
return defValue;
}
/**
* Find out the default DnaNexus project id in the specified configuration file
*
* @return The string value
*/
static String getContextIdByConfig(File config) {
StringBuilder buffer = new StringBuilder();
try {
BufferedReader reader = Files.newBufferedReader(config.toPath(), Charset.defaultCharset());
String line;
while( (line =reader.readLine() ) != null ) {
buffer.append(line).append('\n');
}
JsonNode object = DxJson.parseJson(buffer.toString());
return object.get("DX_PROJECT_CONTEXT_ID").textValue();
}
catch( FileNotFoundException e ) {
throw new IllegalStateException(String.format("Unable to load DnaNexus configuration file: %s -- cannot configure file system", config), e);
}
catch( IOException e ) {
throw new IllegalStateException("Unable to configure DnaNexus file system", e);
}
}
protected DxApi api() {
return api;
}
@Override
public String getScheme() {
return SCHEME;
}
@Override
public Path getPath(URI uri) {
log.trace("Get path by URI: {}", uri);
PathTokens tokens = resolveUri(uri, defaultContextId);
DxFileSystem dxFileSystem = getOrCreateFileSystem(tokens.contextId, tokens.name);
return new DxPath(dxFileSystem, tokens.filePath);
}
protected DxFileSystem newFileSystem() {
return newFileSystem(defaultContextId,defaultContextId);
}
protected DxFileSystem newFileSystem(String contextId, String label) {
if( contextId == null ) { throw new IllegalStateException("Missing 'contextId' attribute"); }
return new DxFileSystem(this, contextId, label);
}
@Override
public final FileSystem newFileSystem(URI uri, Map<String,?> env) {
final String defContextId = getContextIdByMap(env,defaultContextId);
final PathTokens tokens = resolveUri(uri, defContextId);
final String dxContextId = tokens.contextId;
final DxFileSystem result = newFileSystem(dxContextId, tokens.name);
if( fileSystems.putIfAbsent(dxContextId, result) != null ) {
throw new FileSystemAlreadyExistsException();
}
return result;
}
// -- package private
final DxFileSystem getOrCreateFileSystem( String contextId, String name ) {
DxFileSystem dxFileSystem = fileSystems.get(contextId);
if( dxFileSystem == null ) {
log.debug("Creating a new DxFileSystem object with context-id: {}", contextId);
dxFileSystem = newFileSystem(contextId, name);
DxFileSystem former = fileSystems.putIfAbsent(contextId, dxFileSystem);
if( former != null ) {
log.trace("Look ma, got a concurrent creation of a DxFileSystem for context-id: {} -- using the previous instance", contextId);
return former;
}
}
return dxFileSystem;
}
// -- package private
final DxFileSystem getOrCreateFileSystem( String contextId ) {
return getOrCreateFileSystem(contextId,contextId);
}
static class PathTokens {
/** Descriptive name of the context/project */
String name;
/** The real container/project id */
String contextId;
/** The fil path in the container/project */
String filePath;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
PathTokens that = (PathTokens) o;
if (contextId != null ? !contextId.equals(that.contextId) : that.contextId != null) return false;
if (filePath != null ? !filePath.equals(that.filePath) : that.filePath != null) return false;
if (name != null ? !name.equals(that.name) : that.name != null) return false;
return true;
}
@Override
public int hashCode() {
int result = name != null ? name.hashCode() : 0;
result = 31 * result + (contextId != null ? contextId.hashCode() : 0);
result = 31 * result + (filePath != null ? filePath.hashCode() : 0);
return result;
}
}
protected PathTokens checkUri(URI uri) {
Matcher matcher = URI_PATTERN.matcher(uri.toString());
if( !matcher.matches() ) {
throw new IllegalArgumentException("URI does not match this provider: " + uri);
}
String contextId = matcher.group(2);
String path = matcher.group(3);
PathTokens tokens = new PathTokens();
tokens.name = contextId;
tokens.contextId = contextId;
tokens.filePath = path;
return tokens;
}
protected PathTokens resolveUri( URI uri, String defContextId ) {
PathTokens tokens = checkUri(uri);
if( tokens.contextId == null ) {
tokens.contextId = defContextId;
return tokens;
}
Matcher matcher = CONTEXT_ID_PATTERN.matcher(tokens.contextId);
if( !matcher.matches() ) {
// look for this container name by invoking the remote API
try {
List<Map<String,Object>> found = api.projectFind(tokens.contextId);
if( found==null || found.size()!=1 ) {
throw new IllegalStateException(String.format("Unable to retrieve project-id by name: '%s' (1)", tokens.contextId));
}
tokens.contextId = found.get(0).get("id").toString();
}
catch( IOException e ) {
throw new IllegalStateException(String.format("Unable to retrieve project-id by name: '%s' (2)", tokens.contextId), e);
}
}
return tokens;
}
@Override
public final FileSystem getFileSystem(URI uri) {
log.trace("Parsing URI: {}", uri);
PathTokens tokens = resolveUri(uri, defaultContextId);
return fileSystems.get(tokens.contextId);
}
@Override
public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) throws IOException {
return new DxDirectoryStream( toDxPath(dir), filter );
}
public InputStream newInputStream(Path file, OpenOption... options) throws IOException
{
if (options.length > 0) {
for (OpenOption opt: options) {
if (opt != StandardOpenOption.READ)
throw new UnsupportedOperationException("'" + opt + "' not allowed");
}
}
final DxPath path = toDxPath(file);
final String fileId = path.getFileId();
final Map<String,Object> download = api.fileDownload(fileId);
final String url = (String)download.get("url");
final Map<String,String> headers = (Map<String,String>)download.get("headers");
final HttpClient client = DxHttpClient.getInstance().http();
client.getParams().setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
HttpGet get = new HttpGet(url);
get.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.IGNORE_COOKIES);
for( Map.Entry<String,String> item : headers.entrySet() ) {
get.setHeader( item.getKey(), item.getValue());
}
return client.execute(get).getEntity().getContent();
}
private static void checkAllowedOptions( Set<? extends OpenOption> allowed, OpenOption... options ) {
if( options == null ) return;
for( OpenOption opt : options ) {
if( !allowed.contains(opt) ) {
throw new UnsupportedOperationException( opt.toString() + " options not allowed" );
}
}
}
private static Set<? extends OpenOption> OUTPUT_STREAM_VALID_OPTIONS =
new HashSet<>(Arrays.asList(
StandardOpenOption.CREATE,
StandardOpenOption.CREATE_NEW,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING ));
/**
*
* @param path
* @param options
* @return
* @throws IOException
*/
public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException {
checkAllowedOptions(OUTPUT_STREAM_VALID_OPTIONS, options);
// create the file
final DxPath thePath = toDxPath(path);
final DxFileSystem theFileSystem = thePath.getFileSystem();
final String fileId = theFileSystem.fileNew(thePath);
// set the type accordingly
thePath.fileId = fileId;
thePath.type = DxPath.PathType.FILE;
// create the output stream uploaded
return new DxUploadOutputStream(fileId, api);
}
/**
* Operation not supported
*
* @throws UnsupportedOperationException
*/
@Override
public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException
{
throw new UnsupportedOperationException();
}
/**
* Operation not supported
*
* @throws UnsupportedOperationException
*/
@Override
public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
throw new UnsupportedOperationException();
}
/**
* Create a new *remote* folder
* <p>
*
* See https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders%20and%20Deletion#API-method:-/class-xxxx/newFolder
*
* @param path
* @param attrs
* @throws IOException
*/
@Override
public void createDirectory(Path path, FileAttribute<?>... attrs) throws IOException {
if( attrs.length > 0 ) {
throw new UnsupportedOperationException("Attributes on directories are not supported by DnaNexus file system");
}
DxPath dxPath = toDxPath(path);
DxFileSystem theFileSystem = dxPath.getFileSystem();
theFileSystem.createFolder(dxPath);
dxPath.type = DxPath.PathType.DIRECTORY;
}
/**
* Deletes a file. This method works in exactly the manner specified by the
* {@link Files#delete} method.
*
* @param path
* the path to the file to delete
*
* @throws java.nio.file.NoSuchFileException
* if the file does not exist <i>(optional specific exception)</i>
* @throws java.nio.file.DirectoryNotEmptyException
* if the file is a directory and could not otherwise be deleted
* because the directory is not empty <i>(optional specific
* exception)</i>
* <p>
* See http://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method:-/class-xxxx/removeObjects
*
* @param path
* @throws IOException
*/
@Override
public void delete(Path path) throws IOException {
DxPath dxPath = toDxPath(path);
DxFileSystem dxFileSystem = dxPath.getFileSystem();
DxFileAttributes attr = dxPath.readAttributes();
if( attr.isDirectory()) {
dxFileSystem.deleteFolder(dxPath, false);
}
else {
dxFileSystem.deleteFiles(dxPath);
}
// clear all the attributes on this file since does not exist any more
dxPath.clearAttributes();
}
public void deleteDir( DxPath path, boolean recurse ) throws IOException {
path.getFileSystem().deleteFolder(path, recurse);
}
/**
* Implements the *copy* operation using the DnaNexus API *clone*
*
*
* <p>
* See clone https://wiki.dnanexus.com/API-Specification-v1.0.0/Cloning#API-method%3A-%2Fclass-xxxx%2Fclone
*
* @param source
* @param target
* @param options
* @throws IOException
*/
@Override
public void copy(Path source, Path target, CopyOption... options) throws IOException {
List<CopyOption> opts = Arrays.asList(options);
boolean targetExists = Files.exists(target);
if( targetExists ) {
if( Files.isRegularFile(target) ) {
if( opts.contains( StandardCopyOption.REPLACE_EXISTING ) ) {
Files.delete(target);
}
else {
throw new FileAlreadyExistsException("Copy failed -- target file already exists: " + target);
}
}
else if( Files.isDirectory(target)) {
target = target.resolve(source.getFileName());
}
else {
throw new UnsupportedOperationException();
}
}
String name1 = source.getFileName().toString();
String name2 = target.getFileName().toString();
if( !name1.equals(name2) ) {
throw new UnsupportedOperationException("Copy to a file with a different name is not supported: " + source.toString());
}
final DxPath dxSource = toDxPath(source);
final DxFileSystem dxFileSystem = dxSource.getFileSystem();
dxFileSystem.fileCopy( dxSource, toDxPath(target));
}
// TODO move
@Override
public void move(Path source, Path target, CopyOption... options) throws IOException {
// see:
// container move https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method%3A-%2Fclass-xxxx%2Fmove
// container rename https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method%3A-%2Fclass-xxxx%2FrenameFolder
// file rename https://wiki.dnanexus.com/API-Specification-v1.0.0/Name#API-method%3A-%2Fclass-xxxx%2Frename
}
@Override
public boolean isSameFile(Path path, Path path2) throws IOException {
return path.normalize().compareTo(path2.normalize()) == 0;
}
@Override
public boolean isHidden(Path path) throws IOException {
return readAttributes(path, DxFileAttributes.class).isHidden();
}
//TODO getFileStore
@Override
public FileStore getFileStore(Path path) throws IOException {
throw new UnsupportedOperationException();
}
/**
*
* TODO checkAccess
* http://openjdk.java.net/projects/nio/javadoc/java/nio/file/spi/FileSystemProvider.html#checkAccess(java.nio.file.Path, java.nio.file.AccessMode...)
*
* @param path
* @param modes
* @throws IOException
*/
@Override
public void checkAccess(Path path, AccessMode... modes) throws IOException {
toDxPath(path).readAttributes();
}
@Override
@SuppressWarnings("unchecked")
public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
if( type == DxFileAttributeView.class ) {
return (V) new DxFileAttributeView( toDxPath(path) );
}
return null;
}
@Override
@SuppressWarnings("unchecked")
public <V extends BasicFileAttributes> V readAttributes(Path path, Class<V> type, LinkOption... options) throws IOException {
if (type == BasicFileAttributes.class || type == DxFileAttributes.class) {
DxFileAttributeView view = new DxFileAttributeView(toDxPath(path));
return (V)view.readAttributes();
}
return null;
}
@Override
public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
int pos = attributes.indexOf(':');
if (pos != -1) {
String view = attributes.substring(0, pos++);
if( !view.equals( DxFileAttributeView.NAME ) ) {
throw new IllegalArgumentException(String.format("Illegal view for DnaNexus file system: '%s'", view));
}
attributes = attributes.substring(pos);
}
DxFileAttributeView view = new DxFileAttributeView(toDxPath(path));
return view.readAttributes(attributes);
}
@Override
public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {
final DxPath dxPath = toDxPath(path);
final DxFileSystem dxFileSystem = dxPath.getFileSystem();
if (attribute.equals("tags")){
dxFileSystem.fileAddTags(dxPath, (String[]) value);
}
else if (attribute.equals("types")){
dxFileSystem.fileAddTypes(dxPath, (String[]) value);
}
else {
throw new UnsupportedOperationException(String.format("Attribute '%s' cannot be changed", attribute));
}
}
// Checks that the given file is a UnixPath
static final DxPath toDxPath(Path path) {
if (path == null) {
throw new NullPointerException();
}
if (!(path instanceof DxPath)) {
throw new ProviderMismatchException();
}
return (DxPath)path;
}
/**
* @return The current installed instance of the {@code DxFileSystemProvider} or {@code null} if
* no DX provider is installed
*/
static DxFileSystemProvider instance = null;
static DxFileSystemProvider defaultInstance() {
if( instance != null )
return instance;
for (FileSystemProvider provider : FileSystemProvider.installedProviders()) {
if ( provider instanceof DxFileSystemProvider ) {
return instance = (DxFileSystemProvider)provider;
}
}
return null;
}
}