/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/
package alluxio.examples;
import alluxio.AlluxioURI;
import alluxio.client.WriteType;
import alluxio.client.file.FileInStream;
import alluxio.client.file.FileOutStream;
import alluxio.client.file.FileSystem;
import alluxio.client.file.options.CreateFileOptions;
import org.apache.commons.io.IOUtils;
/**
* Example program that demonstrates Alluxio's ability to read and write data across different
* types of storage systems. In particular, this example reads data from S3 and writes data to HDFS.
*
* NOTE: This example assumes that the existence of an Alluxio cluster, the existence of an HDFS
* cluster, and that the properties for accessing S3 (fs.s3n.awsAccessKeyId and fs.s3n
* .awsSecretAccessKey) are set in your Alluxio configuration.
*
* If the above assumptions are met, you can run the example using:
*
* ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>
*/
public final class MultiMount {
private MultiMount() {} // prevent instantiation
/**
* Entry point for the {@link MultiMount} program.
*
* @param args command-line arguments
*/
public static void main(String []args) {
if (args.length != 1) {
System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
System.exit(-1);
}
AlluxioURI mntPath = new AlluxioURI("/mnt");
AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
AlluxioURI hdfsPath = new AlluxioURI(args[0]);
FileSystem fileSystem = FileSystem.Factory.get();
try {
// Make sure mount directory exists.
if (!fileSystem.exists(mntPath)) {
System.out.print("creating " + mntPath + " ... ");
fileSystem.createDirectory(mntPath);
System.out.println("done");
}
// Make sure the S3 mount point does not exist.
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
// Make sure the HDFS mount point does not exist.
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
// Mount S3.
System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
fileSystem.mount(s3Mount, s3Path);
System.out.println("done");
// Mount HDFS.
System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
fileSystem.mount(hdfsMount, hdfsPath);
System.out.println("done");
// Make sure output file does not exist.
if (fileSystem.exists(outputPath)) {
System.out.print("deleting " + outputPath + " ... ");
fileSystem.delete(outputPath);
System.out.println("done");
}
// Open the input stream.
System.out.print("opening " + inputPath + " ... ");
FileInStream is = fileSystem.openFile(inputPath);
System.out.println("done");
// Open the output stream, setting the write type to make sure result is persisted.
System.out.print("opening " + outputPath + " ... ");
CreateFileOptions options =
CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
FileOutStream os = fileSystem.createFile(outputPath, options);
System.out.println("done");
// Copy the data
System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
IOUtils.copy(is, os);
System.out.println("done");
// Close the input stream.
System.out.print("closing " + inputPath + " ... ");
is.close();
System.out.println("done");
// Close the output stream.
System.out.print("closing " + outputPath + " ... ");
os.close();
System.out.println("done");
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
} finally {
// Make sure the S3 mount point is removed.
try {
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
// Make sure the HDFS mount point is removed.
try {
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
}
}
}