package org.lab41.sample.etl.fs;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
/**
* MoveMultiple
*
* @author lab41
*
* Until Oozie supports moving multiple files, this class should do the
* trick
*/
public class MoveMultiple {
private static final String OOZIE_ACTION_OUTPUT_PROPERTIES = "oozie.action.output.properties";
public static void main(String[] args) throws Exception {
String inputPath = args[0];
String outputPath = args[1];
String propKey0 = "status";
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path hadoopInputDir = new Path(inputPath);
Path hadoopOutputDir = new Path(outputPath);
if (!fs.exists(hadoopInputDir)) {
throw new RuntimeException(" Error: " + inputPath
+ " does not exist");
}
if (!fs.exists(hadoopOutputDir)) {
throw new RuntimeException("Error: " + outputPath
+ " does not exist");
}
RemoteIterator<LocatedFileStatus> itr = fs.listFiles(hadoopInputDir,
true);
LocatedFileStatus fileStatus;
int numFiles = 0;
int workingPathLen = fs.getFileStatus(hadoopInputDir).getPath()
.toString().length() + 1; // +1 for trailing slash
String relativePath;
while (itr.hasNext()) {
fileStatus = itr.next();
relativePath = fileStatus.getPath().toString()
.substring(workingPathLen);
Path specificFinalOutput = new Path(hadoopOutputDir
+ Path.SEPARATOR + relativePath);
/**
* if the parent directory does not exist in the destination, then
* create it
*/
if (!fs.isDirectory(specificFinalOutput.getParent())) {
fs.mkdirs(specificFinalOutput.getParent());
}
numFiles++;
System.out.println("Moving " + fileStatus.getPath() + " to "
+ specificFinalOutput);
fs.rename(fileStatus.getPath(), specificFinalOutput);
}
if (numFiles == 0) {
System.out.println("no files..........");
throw new RuntimeException("No Files to process");
}
/**
* Let's let oozie know the number of files that were copied
*/
String oozieProp = System.getProperty(OOZIE_ACTION_OUTPUT_PROPERTIES);
if (oozieProp != null) {
File propFile = new File(oozieProp);
Properties props = new Properties();
props.setProperty(propKey0, "Success " + numFiles + " copied");
OutputStream os = new FileOutputStream(propFile);
props.store(os, "");
os.close();
} else {
System.err
.println("Not using oozie since properties file does not exits");
throw new Exception();
}
}
}