DrillFileSystem.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.drill.exec.store.dfs;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;

import org.apache.drill.exec.ops.OperatorStats;
import org.apache.drill.exec.util.AssertionUtil;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

/**
 * DrillFileSystem is the wrapper around the actual FileSystem implementation.
 *
 * If {@link org.apache.drill.exec.ops.OperatorStats} are provided it returns an instrumented FSDataInputStream to
 * measure IO wait time and tracking file open/close operations.
 */
public class DrillFileSystem extends FileSystem implements OpenFileTracker {
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFileSystem.class);
  private final static boolean TRACKING_ENABLED = AssertionUtil.isAssertionsEnabled();

  public static final String HIDDEN_FILE_PREFIX = "_";
  public static final String DOT_FILE_PREFIX = ".";

  private final ConcurrentMap<DrillFSDataInputStream, DebugStackTrace> openedFiles = Maps.newConcurrentMap();

  private final FileSystem underlyingFs;
  private final OperatorStats operatorStats;
  private final CompressionCodecFactory codecFactory;

  public DrillFileSystem(Configuration fsConf) throws IOException {
    this(fsConf, null);
  }

  public DrillFileSystem(Configuration fsConf, OperatorStats operatorStats) throws IOException {
    this.underlyingFs = FileSystem.get(fsConf);
    this.codecFactory = new CompressionCodecFactory(fsConf);
    this.operatorStats = operatorStats;
  }

  @Override
  public void setConf(Configuration conf) {
    // Guard against setConf(null) call that is called as part of superclass constructor (Configured) of the
    // DrillFileSystem, at which point underlyingFs is null.
    if (conf != null && underlyingFs != null) {
      underlyingFs.setConf(conf);
    }
  }

  @Override
  public Configuration getConf() {
    return underlyingFs.getConf();
  }

  /**
   * If OperatorStats are provided return a instrumented {@link org.apache.hadoop.fs.FSDataInputStream}.
   */
  @Override
  public FSDataInputStream open(Path f, int bufferSize) throws IOException {
    if (operatorStats == null) {
      return underlyingFs.open(f, bufferSize);
    }

    if (TRACKING_ENABLED) {
      DrillFSDataInputStream is = new DrillFSDataInputStream(underlyingFs.open(f, bufferSize), operatorStats, this);
      fileOpened(f, is);
      return is;
    }

    return new DrillFSDataInputStream(underlyingFs.open(f, bufferSize), operatorStats);
  }

  /**
   * If OperatorStats are provided return a instrumented {@link org.apache.hadoop.fs.FSDataInputStream}.
   */
  @Override
  public FSDataInputStream open(Path f) throws IOException {
    if (operatorStats == null) {
      return underlyingFs.open(f);
    }

    if (TRACKING_ENABLED) {
      DrillFSDataInputStream is = new DrillFSDataInputStream(underlyingFs.open(f), operatorStats, this);
      fileOpened(f, is);
      return is;
    }

    return new DrillFSDataInputStream(underlyingFs.open(f), operatorStats);
  }

  @Override
  public void initialize(URI name, Configuration conf) throws IOException {
    underlyingFs.initialize(name, conf);
  }

  @Override
  public String getScheme() {
    return underlyingFs.getScheme();
  }

  @Override
  public FSDataOutputStream create(Path f) throws IOException {
    return underlyingFs.create(f);
  }

  @Override
  public FSDataOutputStream create(Path f, boolean overwrite) throws IOException {
    return underlyingFs.create(f, overwrite);
  }

  @Override
  public FSDataOutputStream create(Path f, Progressable progress) throws IOException {
    return underlyingFs.create(f, progress);
  }

  @Override
  public FSDataOutputStream create(Path f, short replication) throws IOException {
    return underlyingFs.create(f, replication);
  }

  @Override
  public FSDataOutputStream create(Path f, short replication, Progressable progress) throws IOException {
    return underlyingFs.create(f, replication, progress);
  }

  @Override
  public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) throws IOException {
    return underlyingFs.create(f, overwrite, bufferSize);
  }

  @Override
  public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, Progressable progress) throws IOException {
    return underlyingFs.create(f, overwrite, bufferSize, progress);
  }

  @Override
  public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication,
      long blockSize) throws IOException {
    return underlyingFs.create(f, overwrite, bufferSize, replication, blockSize);
  }

  @Override
  public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
    return underlyingFs.create(f, overwrite, bufferSize, replication, blockSize, progress);
  }

  @Override
  public FileStatus getFileStatus(Path f) throws IOException {
    return underlyingFs.getFileStatus(f);
  }

  @Override
  public void createSymlink(Path target, Path link, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException {
    underlyingFs.createSymlink(target, link, createParent);
  }

  @Override
  public FileStatus getFileLinkStatus(Path f) throws AccessControlException, FileNotFoundException,
      UnsupportedFileSystemException, IOException {
    return underlyingFs.getFileLinkStatus(f);
  }

  @Override
  public boolean supportsSymlinks() {
    return underlyingFs.supportsSymlinks();
  }

  @Override
  public Path getLinkTarget(Path f) throws IOException {
    return underlyingFs.getLinkTarget(f);
  }

  @Override
  public FileChecksum getFileChecksum(Path f) throws IOException {
    return underlyingFs.getFileChecksum(f);
  }

  @Override
  public void setVerifyChecksum(boolean verifyChecksum) {
    underlyingFs.setVerifyChecksum(verifyChecksum);
  }

  @Override
  public void setWriteChecksum(boolean writeChecksum) {
    underlyingFs.setWriteChecksum(writeChecksum);
  }

  @Override
  public FsStatus getStatus() throws IOException {
    return underlyingFs.getStatus();
  }

  @Override
  public FsStatus getStatus(Path p) throws IOException {
    return underlyingFs.getStatus(p);
  }

  @Override
  public void setPermission(Path p, FsPermission permission) throws IOException {
    underlyingFs.setPermission(p, permission);
  }

  @Override
  public void setOwner(Path p, String username, String groupname) throws IOException {
    underlyingFs.setOwner(p, username, groupname);
  }

  @Override
  public void setTimes(Path p, long mtime, long atime) throws IOException {
    underlyingFs.setTimes(p, mtime, atime);
  }

  @Override
  public Path createSnapshot(Path path, String snapshotName) throws IOException {
    return underlyingFs.createSnapshot(path, snapshotName);
  }

  @Override
  public void renameSnapshot(Path path, String snapshotOldName, String snapshotNewName) throws IOException {
    underlyingFs.renameSnapshot(path, snapshotOldName, snapshotNewName);
  }

  @Override
  public void deleteSnapshot(Path path, String snapshotName) throws IOException {
    underlyingFs.deleteSnapshot(path, snapshotName);
  }

  @Override
  public void modifyAclEntries(Path path, List<AclEntry> aclSpec) throws IOException {
    underlyingFs.modifyAclEntries(path, aclSpec);
  }

  @Override
  public void removeAclEntries(Path path, List<AclEntry> aclSpec) throws IOException {
    underlyingFs.removeAclEntries(path, aclSpec);
  }

  @Override
  public void removeDefaultAcl(Path path) throws IOException {
    underlyingFs.removeDefaultAcl(path);
  }

  @Override
  public void removeAcl(Path path) throws IOException {
    underlyingFs.removeAcl(path);
  }

  @Override
  public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
    underlyingFs.setAcl(path, aclSpec);
  }

  @Override
  public AclStatus getAclStatus(Path path) throws IOException {
    return underlyingFs.getAclStatus(path);
  }

  @Override
  public Path getWorkingDirectory() {
    return underlyingFs.getWorkingDirectory();
  }

  @Override
  public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException {
    return underlyingFs.append(f, bufferSize, progress);
  }

  @Override
  public void concat(Path trg, Path[] psrcs) throws IOException {
    underlyingFs.concat(trg, psrcs);
  }

  @Override
  @Deprecated
  public short getReplication(Path src) throws IOException {
    return underlyingFs.getReplication(src);
  }

  @Override
  public boolean setReplication(Path src, short replication) throws IOException {
    return underlyingFs.setReplication(src, replication);
  }

  @Override
  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
    return underlyingFs.mkdirs(f, permission);
  }

  @Override
  public void copyFromLocalFile(Path src, Path dst) throws IOException {
    underlyingFs.copyFromLocalFile(src, dst);
  }

  @Override
  public void moveFromLocalFile(Path[] srcs, Path dst) throws IOException {
    underlyingFs.moveFromLocalFile(srcs, dst);
  }

  @Override
  public void moveFromLocalFile(Path src, Path dst) throws IOException {
    underlyingFs.moveFromLocalFile(src, dst);
  }

  @Override
  public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
    underlyingFs.copyFromLocalFile(delSrc, src, dst);
  }

  @Override
  public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException {
    underlyingFs.copyFromLocalFile(delSrc, overwrite, srcs, dst);
  }

  @Override
  public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException {
    underlyingFs.copyFromLocalFile(delSrc, overwrite, src, dst);
  }

  @Override
  public void copyToLocalFile(Path src, Path dst) throws IOException {
    underlyingFs.copyToLocalFile(src, dst);
  }

  @Override
  public void moveToLocalFile(Path src, Path dst) throws IOException {
    underlyingFs.moveToLocalFile(src, dst);
  }

  @Override
  public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
    underlyingFs.copyToLocalFile(delSrc, src, dst);
  }

  @Override
  public void copyToLocalFile(boolean delSrc, Path src, Path dst, boolean useRawLocalFileSystem) throws IOException {
    underlyingFs.copyToLocalFile(delSrc, src, dst, useRawLocalFileSystem);
  }

  @Override
  public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException {
    return underlyingFs.startLocalOutput(fsOutputFile, tmpLocalFile);
  }

  @Override
  public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException {
    underlyingFs.completeLocalOutput(fsOutputFile, tmpLocalFile);
  }

  @Override
  public void close() throws IOException {
    if (TRACKING_ENABLED) {
      if (openedFiles.size() != 0) {
        final StringBuffer errMsgBuilder = new StringBuffer();

        errMsgBuilder.append(String.format("Not all files opened using this FileSystem are closed. " + "There are" +
            " still [%d] files open.\n", openedFiles.size()));

        for (DebugStackTrace stackTrace : openedFiles.values()) {
          stackTrace.addToStringBuilder(errMsgBuilder);
        }

        final String errMsg = errMsgBuilder.toString();
        logger.error(errMsg);
        throw new IllegalStateException(errMsg);
      }
    }
  }

  @Override
  public long getUsed() throws IOException {
    return underlyingFs.getUsed();
  }

  @Override
  @Deprecated
  public long getBlockSize(Path f) throws IOException {
    return underlyingFs.getBlockSize(f);
  }

  @Override
  @Deprecated
  public long getDefaultBlockSize() {
    return underlyingFs.getDefaultBlockSize();
  }

  @Override
  public long getDefaultBlockSize(Path f) {
    return underlyingFs.getDefaultBlockSize(f);
  }

  @Override
  @Deprecated
  public short getDefaultReplication() {
    return underlyingFs.getDefaultReplication();
  }

  @Override
  public short getDefaultReplication(Path path) {
    return underlyingFs.getDefaultReplication(path);
  }

  @Override
  public boolean mkdirs(Path folderPath) throws IOException {
    if (!underlyingFs.exists(folderPath)) {
      return underlyingFs.mkdirs(folderPath);
    } else if (!underlyingFs.getFileStatus(folderPath).isDir()) {
      throw new IOException("The specified folder path exists and is not a folder.");
    }
    return false;
  }

  @Override
  public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize,
      short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException {
    return underlyingFs.create(f, permission, flags, bufferSize, replication, blockSize, progress, checksumOpt);
  }

  @Override
  @Deprecated
  public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication,
      long blockSize, Progressable progress) throws IOException {
    return underlyingFs.createNonRecursive(f, overwrite, bufferSize, replication, blockSize, progress);
  }

  @Override
  @Deprecated
  public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize,
      short replication, long blockSize, Progressable progress) throws IOException {
    return underlyingFs.createNonRecursive(f, permission, overwrite, bufferSize, replication, blockSize, progress);
  }

  @Override
  @Deprecated
  public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
    return underlyingFs.createNonRecursive(f, permission, flags, bufferSize, replication, blockSize, progress);
  }

  @Override
  public boolean createNewFile(Path f) throws IOException {
    return underlyingFs.createNewFile(f);
  }

  @Override
  public FSDataOutputStream append(Path f) throws IOException {
    return underlyingFs.append(f);
  }

  @Override
  public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
    return underlyingFs.append(f, bufferSize);
  }

  @Override
  public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short
      replication, long blockSize, Progressable progress) throws IOException {
    return underlyingFs.create(f, permission, overwrite, bufferSize, replication, blockSize, progress);
  }

  @Override
  public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize,
      short replication, long blockSize, Progressable progress) throws IOException {
    return underlyingFs.create(f, permission, flags, bufferSize, replication, blockSize, progress);
  }

  @Override
  public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
    return underlyingFs.listStatus(f);
  }

  @Override
  public RemoteIterator<Path> listCorruptFileBlocks(Path path) throws IOException {
    return underlyingFs.listCorruptFileBlocks(path);
  }

  @Override
  public FileStatus[] listStatus(Path f, PathFilter filter) throws FileNotFoundException, IOException {
    return underlyingFs.listStatus(f, filter);
  }

  @Override
  public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException {
    return underlyingFs.listStatus(files);
  }

  @Override
  public FileStatus[] listStatus(Path[] files, PathFilter filter) throws FileNotFoundException, IOException {
    return underlyingFs.listStatus(files, filter);
  }

  @Override
  public FileStatus[] globStatus(Path pathPattern) throws IOException {
    return underlyingFs.globStatus(pathPattern);
  }

  @Override
  public FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException {
    return underlyingFs.globStatus(pathPattern, filter);
  }

  @Override
  public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f) throws FileNotFoundException, IOException {
    return underlyingFs.listLocatedStatus(f);
  }

  @Override
  public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive) throws FileNotFoundException, IOException {
    return underlyingFs.listFiles(f, recursive);
  }

  @Override
  public Path getHomeDirectory() {
    return underlyingFs.getHomeDirectory();
  }

  @Override
  public void setWorkingDirectory(Path new_dir) {
    underlyingFs.setWorkingDirectory(new_dir);
  }

  @Override
  public boolean rename(Path src, Path dst) throws IOException {
    return underlyingFs.rename(src, dst);
  }

  @Override
  @Deprecated
  public boolean delete(Path f) throws IOException {
    return underlyingFs.delete(f);
  }

  @Override
  public boolean delete(Path f, boolean recursive) throws IOException {
    return underlyingFs.delete(f, recursive);
  }

  @Override
  public boolean deleteOnExit(Path f) throws IOException {
    return underlyingFs.deleteOnExit(f);
  }

  @Override
  public boolean cancelDeleteOnExit(Path f) {
    return underlyingFs.cancelDeleteOnExit(f);
  }

  @Override
  public boolean exists(Path f) throws IOException {
    return underlyingFs.exists(f);
  }

  @Override
  public boolean isDirectory(Path f) throws IOException {
    return underlyingFs.isDirectory(f);
  }

  @Override
  public boolean isFile(Path f) throws IOException {
    return underlyingFs.isFile(f);
  }

  @Override
  @Deprecated
  public long getLength(Path f) throws IOException {
    return underlyingFs.getLength(f);
  }

  @Override
  public ContentSummary getContentSummary(Path f) throws IOException {
    return underlyingFs.getContentSummary(f);
  }

  @Override
  public URI getUri() {
    return underlyingFs.getUri();
  }

  @Override
  @LimitedPrivate({"HDFS", "MapReduce"})
  public String getCanonicalServiceName() {
    return underlyingFs.getCanonicalServiceName();
  }

  @Override
  @Deprecated
  public String getName() {
    return underlyingFs.getName();
  }

  @Override
  public Path makeQualified(Path path) {
    return underlyingFs.makeQualified(path);
  }

  @Override
  @Private
  public Token<?> getDelegationToken(String renewer) throws IOException {
    return underlyingFs.getDelegationToken(renewer);
  }

  @Override
  @LimitedPrivate({"HDFS", "MapReduce"})
  public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
    return underlyingFs.addDelegationTokens(renewer, credentials);
  }

  @Override
  @LimitedPrivate({"HDFS"})
  @VisibleForTesting
  public FileSystem[] getChildFileSystems() {
    return underlyingFs.getChildFileSystems();
  }

  @Override
  public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {
    return underlyingFs.getFileBlockLocations(file, start, len);
  }

  @Override
  public BlockLocation[] getFileBlockLocations(Path p, long start, long len) throws IOException {
    return underlyingFs.getFileBlockLocations(p, start, len);
  }

  @Override
  @Deprecated
  public FsServerDefaults getServerDefaults() throws IOException {
    return underlyingFs.getServerDefaults();
  }

  @Override
  public FsServerDefaults getServerDefaults(Path p) throws IOException {
    return underlyingFs.getServerDefaults(p);
  }

  @Override
  public Path resolvePath(Path p) throws IOException {
    return underlyingFs.resolvePath(p);
  }

  @Override
  public boolean truncate(final Path f, final long newLength) throws IOException {
    return underlyingFs.truncate(f, newLength);
  }

  @Override
  public RemoteIterator<FileStatus> listStatusIterator(final Path p) throws FileNotFoundException, IOException {
    return underlyingFs.listStatusIterator(p);
  }

  @Override
  public void access(final Path path, final FsAction mode) throws AccessControlException, FileNotFoundException, IOException {
    underlyingFs.access(path, mode);
  }

  @Override
  public FileChecksum getFileChecksum(final Path f, final long length) throws IOException {
    return underlyingFs.getFileChecksum(f, length);
  }

  @Override
  public void setXAttr(final Path path, final String name, final byte[] value) throws IOException {
    underlyingFs.setXAttr(path, name, value);
  }

  @Override
  public void setXAttr(final Path path, final String name, final byte[] value, final EnumSet<XAttrSetFlag> flag) throws IOException {
    underlyingFs.setXAttr(path, name, value, flag);
  }

  @Override
  public byte[] getXAttr(final Path path, final String name) throws IOException {
    return underlyingFs.getXAttr(path, name);
  }

  @Override
  public Map<String, byte[]> getXAttrs(final Path path) throws IOException {
    return underlyingFs.getXAttrs(path);
  }

  @Override
  public Map<String, byte[]> getXAttrs(final Path path, final List<String> names) throws IOException {
    return underlyingFs.getXAttrs(path, names);
  }

  @Override
  public List<String> listXAttrs(final Path path) throws IOException {
    return underlyingFs.listXAttrs(path);
  }

  @Override
  public void removeXAttr(final Path path, final String name) throws IOException {
    underlyingFs.removeXAttr(path, name);
  }

  public List<FileStatus> list(boolean recursive, Path... paths) throws IOException {
    if (recursive) {
      List<FileStatus> statuses = Lists.newArrayList();
      for (Path p : paths) {
        addRecursiveStatus(underlyingFs.getFileStatus(p), statuses);
      }
      return statuses;

    } else {
      return Lists.newArrayList(underlyingFs.listStatus(paths));
    }
  }

  private void addRecursiveStatus(FileStatus parent, List<FileStatus> listToFill) throws IOException {
    if (parent.isDir()) {
      Path pattern = new Path(parent.getPath(), "*");
      FileStatus[] sub = underlyingFs.globStatus(pattern, new DrillPathFilter());
      for(FileStatus s : sub){
        if (s.isDir()) {
          addRecursiveStatus(s, listToFill);
        } else {
          listToFill.add(s);
        }
      }
    } else {
      listToFill.add(parent);
    }
  }

  public InputStream openPossiblyCompressedStream(Path path) throws IOException {
    CompressionCodec codec = codecFactory.getCodec(path); // infers from file ext.
    if (codec != null) {
      return codec.createInputStream(open(path));
    } else {
      return open(path);
    }
  }
  @Override
  public void fileOpened(Path path, DrillFSDataInputStream fsDataInputStream) {
    openedFiles.put(fsDataInputStream, new DebugStackTrace(path, Thread.currentThread().getStackTrace()));
  }

  @Override
  public void fileClosed(DrillFSDataInputStream fsDataInputStream) {
    openedFiles.remove(fsDataInputStream);
  }

  public static class DebugStackTrace {
    final private StackTraceElement[] elements;
    final private Path path;

    public DebugStackTrace(Path path, StackTraceElement[] elements) {
      this.path = path;
      this.elements = elements;
    }

    public void addToStringBuilder(StringBuffer sb) {
      sb.append("File '");
      sb.append(path.toString());
      sb.append("' opened at callstack:\n");

      // add all stack elements except the top three as they point to DrillFileSystem.open() and inner stack elements.
      for (int i = 3; i < elements.length; i++) {
        sb.append("\t");
        sb.append(elements[i]);
        sb.append("\n");
      }
      sb.append("\n");
    }
  }
}