/*
* Copyright (C) 2014 Indeed Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.indeed.imhotep.archive;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author jplaisance
*/
public final class CopyIndexTimeRangeToLocal {
static {
DateTimeZone.setDefault(DateTimeZone.forOffsetHours(-6));
}
public static void main(String[] args) throws IOException {
if (args.length < 4) {
System.err.println("ARGS: from to start end");
System.exit(1);
}
System.out.println("Copying archives from " + args[0] + " to " + args[1]);
copy(new Path(args[0]), new File(args[1]), new DateTime(args[2]), new DateTime(args[3]));
System.out.println("Wrote to: " + args[1]);
}
public static void copy(Path from, File to, DateTime start, DateTime end) throws IOException {
copy(from.getFileSystem(new Configuration()), from, to, start, end);
}
private static final Pattern shardPattern = Pattern.compile("index([0-9]{4})([0-9]{2})([0-9]{2})\\.([0-9]{2})\\.[0-9]{14}\\.sqar");
public static void copy(FileSystem fs, Path from, File to, final DateTime start, final DateTime end) throws IOException {
if ((to.exists() && !to.isDirectory())) {
throw new FileNotFoundException(to.getAbsolutePath() + " is not a directory");
} else if (!to.exists() && !to.mkdirs()) {
throw new IOException("unable to create directory " + to.getAbsolutePath());
}
for (final FileStatus status : fs.listStatus(from, new PathFilter() {
public boolean accept(final Path path) {
final Matcher m = shardPattern.matcher(path.getName());
if (!m.matches()) {
return false;
}
final DateTime dateTime = new DateTime(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)), Integer.parseInt(m.group(3)), Integer.parseInt(m.group(4)), 0);
return dateTime.getMillis() >= start.getMillis() && dateTime.getMillis() < end.getMillis();
}
})) {
if (status.isDir()) {
final Path path = status.getPath();
System.out.println("copying " + path);
final String pathName = path.getName();
final String dirName = pathName.endsWith(".sqar") ? pathName.substring(0, pathName.length() - 5) : pathName;
new SquallArchiveReader(fs, path).copyAllToLocal(new File(to, dirName));
}
}
}
}