/*
* Copyright 2013 Cloudera Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.kitesdk.data.spi.filesystem;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
import org.kitesdk.compat.DynMethods;
import org.kitesdk.data.DatasetDescriptor;
import org.kitesdk.data.DatasetIOException;
import org.kitesdk.data.Format;
import org.kitesdk.data.Formats;
import org.kitesdk.data.IncompatibleSchemaException;
import org.kitesdk.data.PartitionStrategy;
import org.kitesdk.data.ValidationException;
import org.kitesdk.data.spi.Pair;
import org.kitesdk.data.spi.SchemaUtil;
import org.kitesdk.data.spi.Schemas;
import org.kitesdk.data.spi.partition.ProvidedFieldPartitioner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FileSystemUtil {
private static final Logger LOG = LoggerFactory.getLogger(FileSystemUtil.class);
private static final List<Format> SUPPORTED_FORMATS = Lists.newArrayList(
Formats.AVRO, Formats.PARQUET, Formats.JSON, Formats.CSV);
/**
* Creates, if necessary, the given the location for {@code descriptor}.
*
* @param conf A Configuration
* @param descriptor A DatasetDescriptor
* @throws DatasetIOException
* @since 0.13.0
*/
public static void ensureLocationExists(
DatasetDescriptor descriptor, Configuration conf) {
Preconditions.checkNotNull(descriptor.getLocation(),
"Cannot get FileSystem for a descriptor with no location");
Path dataPath = new Path(descriptor.getLocation().toString());
FileSystem fs = null;
try {
fs = dataPath.getFileSystem(conf);
} catch (IOException e) {
throw new DatasetIOException(
"Cannot get FileSystem for descriptor: " + descriptor, e);
}
try {
if (!fs.exists(dataPath)) {
fs.mkdirs(dataPath);
}
} catch (IOException e) {
throw new DatasetIOException("Cannot access data location", e);
}
}
static List<Pair<Path, Path>> stageMove(FileSystem fs, Path src,
Path dest, String ext) {
List<Pair<Path, Path>> staged;
try {
// make sure the destination exists
if (!fs.exists(dest)) {
fs.mkdirs(dest);
}
FileStatus[] stats = fs.listStatus(src);
staged = Lists.newArrayList();
for (FileStatus stat : stats) {
if (stat.isDir()) {
continue;
}
Path srcFile = stat.getPath();
Path dotFile = new Path(dest, "." + srcFile.getName() + "." + ext);
Path destFile = new Path(dest, srcFile.getName());
if (fs.rename(srcFile, dotFile)) {
staged.add(Pair.of(dotFile, destFile));
} else {
throw new IOException(
"Failed to rename " + srcFile + " to " + dotFile);
}
}
} catch (IOException e) {
throw new DatasetIOException(
"Could not move contents of " + src + " to " + dest, e);
}
return staged;
}
static void finishMove(FileSystem fs, List<Pair<Path, Path>> staged) {
try {
for (Pair<Path, Path> pair : staged) {
if (!fs.rename(pair.first(), pair.second())) {
throw new IOException(
"Failed to rename " + pair.first() + " to " + pair.second());
}
}
} catch (IOException e) {
throw new DatasetIOException("Could not finish replacement", e);
}
}
/**
* Replace {@code destination} with {@code replacement}.
* <p>
* If this method fails in any step, recover using these steps:
* <ol>
* <li>If {@code .name.replacement} exists, but {@code name} does not, move
* it to {@code name}</li>
* <li>If {@code .name.replacement} and {@code name} exist, run this method
* again with the same list of additional removals</li>
* </ol>
*
* @param fs the FileSystem
* @param destination a Path
* @param replacement a Path that replaces the destination
* @param removals a List of paths that should also be removed
*/
static void replace(FileSystem fs, Path root, Path destination,
Path replacement, List<Path> removals) {
try {
// Ensure the destination exists because it acts as a recovery signal. If
// the directory exists, then recovery must go through the entire
// replacement process again. If it does not, then the dir can be moved.
if (!fs.exists(destination)) {
fs.mkdirs(destination);
}
Path staged = new Path(destination.getParent(),
"." + destination.getName() + ".replacement");
// First move into the destination folder to ensure moves work. It is
// okay to run this method on the staged path
if (!staged.equals(replacement) && !fs.rename(replacement, staged)) {
throw new IOException(
"Failed to rename " + replacement + " to " + staged);
}
// Remove any additional directories included in the replacement. This
// handles the case where there are multiple directories for the same
// logical partition. For example, dataset/a=2/ and dataset/2/
for (Path toRemove : removals) {
if (toRemove.equals(destination)) {
// destination is deleted last
continue;
}
FileSystemUtil.cleanlyDelete(fs, root, toRemove, false);
}
// remove the directory that will be replaced with a move
fs.delete(destination, true /* recursively */ );
// move the replacement to the final location
if (!fs.rename(staged, destination)) {
throw new IOException(
"Failed to rename " + staged + " to " + destination);
}
} catch (IOException e) {
throw new DatasetIOException(
"Could not replace " + destination + " with " + replacement, e);
}
}
static boolean cleanlyDelete(FileSystem fs, Path root, Path path) {
return cleanlyDelete(fs, root, path, false);
}
static boolean cleanlyMoveToTrash(FileSystem fs, Path root, Path path) {
return cleanlyDelete(fs, root, path, true);
}
private static boolean cleanlyDelete(FileSystem fs, Path root, Path path, boolean useTrash) {
Preconditions.checkNotNull(fs, "File system cannot be null");
Preconditions.checkNotNull(root, "Root path cannot be null");
Preconditions.checkNotNull(path, "Path to delete cannot be null");
try {
boolean deleted;
// attempt to relativize the path to delete
Path relativePath;
if (path.isAbsolute()) {
relativePath = new Path(root.toUri().relativize(path.toUri()));
} else {
relativePath = path;
}
if (relativePath.isAbsolute()) {
// path is not relative to the root. delete just the path
LOG.debug("Deleting path {}", path);
deleted = useTrash ? Trash.moveToAppropriateTrash(fs, path, fs.getConf())
: fs.delete(path, true /* include any files */);
} else {
// the is relative to the root path
Path absolute = new Path(root, relativePath);
LOG.debug("Deleting path {}", absolute);
deleted = useTrash ? Trash.moveToAppropriateTrash(fs, absolute, fs.getConf())
: fs.delete(absolute, true /* include any files */);
// iterate up to the root, removing empty directories
for (Path current = absolute.getParent();
!current.equals(root) && !(current.getParent() == null);
current = current.getParent()) {
final FileStatus[] stats = fs.listStatus(current);
if (stats == null || stats.length == 0) {
// dir is empty and should be removed
LOG.debug("Deleting empty path {}", current);
deleted = fs.delete(current, true) || deleted;
} else {
// all parent directories will be non-empty
break;
}
}
}
return deleted;
} catch (IOException ex) {
throw new DatasetIOException("Could not cleanly delete path:" + path, ex);
}
}
public static Schema schema(String name, FileSystem fs, Path location) throws IOException {
if (!fs.exists(location)) {
return null;
}
return visit(new GetSchema(name), fs, location);
}
public static PartitionStrategy strategy(FileSystem fs, Path location) throws IOException {
if (!fs.exists(location)) {
return null;
}
List<Pair<String, Class<? extends Comparable>>> pairs = visit(
new GetPartitionInfo(), fs, location);
if (pairs == null || pairs.isEmpty() || pairs.size() <= 1) {
return null;
}
PartitionStrategy.Builder builder = new PartitionStrategy.Builder();
// skip the initial partition because it is the containing directory
for (int i = 1; i < pairs.size(); i += 1) {
Pair<String, Class<? extends Comparable>> pair = pairs.get(i);
builder.provided(
pair.first() == null ? "partition_" + i : pair.first(),
ProvidedFieldPartitioner.valuesString(pair.second()));
}
return builder.build();
}
public static Format format(FileSystem fs, Path location) throws IOException {
if (!fs.exists(location)) {
return null;
}
return visit(new GetFormat(), fs, location);
}
private static abstract class PathVisitor<T> {
abstract T directory(FileSystem fs, Path path, List<T> children) throws IOException;
abstract T file(FileSystem fs, Path path) throws IOException;
}
private static <T> T visit(PathVisitor<T> visitor, FileSystem fs, Path path)
throws IOException {
return visit(visitor, fs, path, Lists.<Path>newArrayList());
}
private static final DynMethods.UnboundMethod IS_SYMLINK;
static {
DynMethods.UnboundMethod isSymlink;
try {
isSymlink = new DynMethods.Builder("isSymlink")
.impl(FileStatus.class)
.buildChecked();
} catch (NoSuchMethodException e) {
isSymlink = null;
}
IS_SYMLINK = isSymlink;
}
private static <T> T visit(PathVisitor<T> visitor, FileSystem fs, Path path,
List<Path> followedLinks) throws IOException {
if (fs.isFile(path)) {
return visitor.file(fs, path);
} else if (IS_SYMLINK != null &&
IS_SYMLINK.<Boolean>invoke(fs.getFileStatus(path))) {
Preconditions.checkArgument(!followedLinks.contains(path),
"Encountered recursive path structure at link: " + path);
followedLinks.add(path); // no need to remove
return visit(visitor, fs, fs.getLinkTarget(path), followedLinks);
}
List<T> children = Lists.newArrayList();
FileStatus[] statuses = fs.listStatus(path, PathFilters.notHidden());
for (FileStatus stat : statuses) {
children.add(visit(visitor, fs, stat.getPath()));
}
return visitor.directory(fs, path, children);
}
private static class GetPartitionInfo
extends PathVisitor<List<Pair<String, Class<? extends Comparable>>>> {
private static final Splitter EQUALS = Splitter.on('=').limit(2).trimResults();
@Override
List<Pair<String, Class<? extends Comparable>>> directory(
FileSystem fs, Path path, List<List<Pair<String, Class<? extends Comparable>>>> children)
throws IOException {
// merge the levels under this one
List<Pair<String, Class<? extends Comparable>>> accumulated = Lists.newArrayList();
for (List<Pair<String, Class<? extends Comparable>>> child : children) {
if (child == null) {
continue;
}
for (int i = 0; i < child.size(); i += 1) {
if (accumulated.size() > i) {
Pair<String, Class<? extends Comparable>> pair = merge(
accumulated.get(i), child.get(i));
accumulated.set(i, pair);
} else if (child.get(i) != null) {
accumulated.add(child.get(i));
}
}
}
List<String> parts = Lists.newArrayList(EQUALS.split(path.getName()));
String name;
String value;
if (parts.size() == 2) {
name = parts.get(0);
value = parts.get(1);
} else {
name = null;
value = parts.get(0);
}
accumulated.add(0,
new Pair<String, Class<? extends Comparable>>(name, dataClass(value)));
return accumulated;
}
@Override
List<Pair<String, Class<? extends Comparable>>> file(
FileSystem fs, Path path) throws IOException {
return null;
}
public Pair<String, Class<? extends Comparable>> merge(
Pair<String, Class<? extends Comparable>> left,
Pair<String, Class<? extends Comparable>> right) {
String name = left.first();
if (name == null || name.isEmpty()) {
name = right.first();
}
if (left.second() == String.class) {
return new Pair<String, Class<? extends Comparable>>(name, String.class);
} else if (right.second() == String.class) {
return new Pair<String, Class<? extends Comparable>>(name, String.class);
} else if (left.second() == Long.class) {
return new Pair<String, Class<? extends Comparable>>(name, Long.class);
} else if (right.second() == Long.class) {
return new Pair<String, Class<? extends Comparable>>(name, Long.class);
}
return new Pair<String, Class<? extends Comparable>>(name, Integer.class);
}
public Class<? extends Comparable> dataClass(String value) {
try {
Integer.parseInt(value);
return Integer.class;
} catch (NumberFormatException e) {
// not an integer
}
try {
Long.parseLong(value);
return Long.class;
} catch (NumberFormatException e) {
// not a long
}
return String.class;
}
}
private static class GetFormat extends PathVisitor<Format> {
@Override
Format directory(FileSystem fs, Path path, List<Format> formats) throws IOException {
Format format = null;
for (Format otherFormat : formats) {
if (format == null) {
format = otherFormat;
} else if (!format.equals(otherFormat)) {
throw new ValidationException(String.format(
"Path contains multiple formats (%s, %s): %s",
format, otherFormat, path));
}
}
return format;
}
@Override
Format file(FileSystem fs, Path path) throws IOException {
return formatFromExt(path);
}
}
private static class GetSchema extends PathVisitor<Schema> {
private final String name;
public GetSchema(String name) {
this.name = name;
}
@Override
Schema directory(FileSystem fs, Path path, List<Schema> schemas) {
Schema merged = null;
for (Schema schema : schemas) {
merged = merge(merged, schema);
}
return merged;
}
@Override
Schema file(FileSystem fs, Path path) throws IOException {
String filename = path.getName();
if (filename.endsWith(Formats.AVRO.getExtension())) {
return Schemas.fromAvro(fs, path);
} else if (filename.endsWith(Formats.PARQUET.getExtension())) {
return Schemas.fromParquet(fs, path);
} else if (filename.endsWith(Formats.JSON.getExtension())) {
return Schemas.fromJSON(name, fs, path);
}
return null;
}
}
/**
* Finds potential datasets by crawling a directory tree.
* <p>
* This method looks for any data files and directories appear to form a
* dataset. This deliberately ignores information that may be stored in the
* Hive metastore or .metadata folders.
* <p>
* Recognizes only Avro, Parquet, and JSON data files.
*
* @param fs a FileSystem for the root path
* @param path a root Path that will be searched
* @return a Collection with a DatasetDescriptor for each potential dataset.
* @throws IOException
*/
public static Collection<DatasetDescriptor> findPotentialDatasets(
FileSystem fs, Path path) throws IOException {
List<DatasetDescriptor> descriptors = Lists.newArrayList();
Result result = visit(new FindDatasets(), fs, path);
if (result instanceof Result.Table) {
descriptors.add(descriptor(fs, (Result.Table) result));
} else if (result instanceof Result.Group) {
for (Result.Table table : ((Result.Group) result).tables) {
descriptors.add(descriptor(fs, table));
}
}
return descriptors;
}
private static DatasetDescriptor descriptor(FileSystem fs, Result.Table table)
throws IOException {
// inspect the path to determine the partition strategy
PartitionStrategy strategy = strategy(fs, table.location);
DatasetDescriptor.Builder builder = new DatasetDescriptor.Builder()
.format(table.format)
.schema(table.schema)
.partitionStrategy(strategy)
.location(table.location);
if (table.depth < 0) {
builder.property("kite.filesystem.mixed-depth", "true");
}
return builder.build();
}
private interface Result {
// An unknown data path
class Unknown implements Result {
}
// A table of data
class Table implements Result {
private static final int UNKNOWN_DEPTH = 0;
private static final int MIXED_DEPTH = -1;
private final Path location;
private final Format format;
private final Schema schema;
private final int depth;
public Table(Path location, Format format, Schema schema, int depth) {
this.location = location;
this.format = format;
this.schema = schema;
this.depth = depth;
}
}
// A group of tables
class Group implements Result {
private final List<Table> tables;
private final boolean containsUnknown;
public Group(List<Table> tables, boolean containsUnknown) {
this.tables = tables;
this.containsUnknown = containsUnknown;
}
}
}
private static class FindDatasets extends PathVisitor<Result> {
@Override
Result directory(FileSystem fs, Path path, List<Result> children) throws IOException {
// there are two possible outcomes for this method:
// 1. all child tables are compatible and part of one dataset (Table)
// 2. each valid child is a separate dataset (Group)
boolean allCompatible = true; // assume compatible to start
boolean containsUnknown = false;
// if all are compatible
Schema mergedSchema = null;
Format onlyFormat = null;
int depth = Result.Table.UNKNOWN_DEPTH;
// if all are separate datasets
List<Result.Table> tables = Lists.newArrayList();
for (Result child : children) {
if (child instanceof Result.Unknown) {
// not compatible at this level because a data file is not supported
allCompatible = false;
containsUnknown = true;
} else if (child instanceof Result.Group) {
// not compatible at a lower level
Result.Group group = (Result.Group) child;
containsUnknown |= group.containsUnknown;
if (containsUnknown || !group.tables.isEmpty()) {
// not compatible if there was an unknown or was not empty
allCompatible = false;
}
tables.addAll(group.tables);
} else {
Result.Table table = (Result.Table) child;
tables.add(table); // always add table in case not compatible later
// if all tables are currently compatible, add the latest table
if (allCompatible) {
try {
mergedSchema = merge(mergedSchema, table.schema);
} catch (IncompatibleSchemaException e) {
allCompatible = false;
}
if (onlyFormat == null) {
onlyFormat = table.format;
} else if (onlyFormat != table.format) {
allCompatible = false;
}
if (depth == Result.Table.UNKNOWN_DEPTH) {
depth = table.depth;
} else if (depth != table.depth) {
depth = Result.Table.MIXED_DEPTH;
}
}
}
}
if (allCompatible && tables.size() > 0) {
if (tables.size() == 1) {
// only one, use the existing location rather than higher up the path
return tables.get(0);
} else {
// more than one, use the path at this level
return new Result.Table(path, onlyFormat, mergedSchema, depth);
}
} else {
return new Result.Group(tables, containsUnknown);
}
}
@Override
Result file(FileSystem fs, Path path) throws IOException {
Format format = formatFromExt(path);
Schema schema = null;
if (format == Formats.AVRO) {
schema = Schemas.fromAvro(fs, path);
} else if (format == Formats.PARQUET) {
schema = Schemas.fromParquet(fs, path);
} else if (format == Formats.JSON) {
schema = Schemas.fromJSON("record", fs, path);
}
if (schema == null) {
return new Result.Unknown();
}
return new Result.Table(path, format, schema, path.depth());
}
}
private static final Splitter DOT = Splitter.on('.');
private static Format formatFromExt(Path path) {
String filename = path.getName();
String ext = Iterables.getLast(DOT.split(filename));
for (Format format : SUPPORTED_FORMATS) {
if (ext.equals(format.getExtension())) {
return format;
}
}
return null;
}
private static Schema merge(@Nullable Schema left, @Nullable Schema right) {
if (left == null) {
return right;
} else if (right == null) {
return left;
} else {
return SchemaUtil.merge(left, right);
}
}
/**
* Determine whether a FileSystem that supports efficient file renaming is being used. Two known
* FileSystem implementations that currently lack this feature are S3N and S3A.
*
* @param fsUri the FileSystem URI
* @param conf the FileSystem Configuration
* @return {@code true} if the FileSystem URI or {@link FileSystemProperties#SUPPORTS_RENAME_PROP
* configuration override} indicates that the FileSystem implementation supports efficient
* rename operations, {@code false} otherwise.
*/
public static boolean supportsRename(URI fsUri, Configuration conf) {
String fsUriScheme = fsUri.getScheme();
// Only S3 is known to not support renaming, but allow configuration override.
// This logic is intended as a temporary placeholder solution and should
// be revisited once HADOOP-9565 has been completed.
return conf.getBoolean(FileSystemProperties.SUPPORTS_RENAME_PROP,
!(fsUriScheme.equalsIgnoreCase("s3n") || fsUriScheme.equalsIgnoreCase("s3a")));
}
}