TestFileStatusExtended.java

package org.apache.hadoop.hdfs;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Random;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.namenode.NameNode;

import org.junit.After;
import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.Test;

public class TestFileStatusExtended {
  private static MiniDFSCluster cluster;
  private static Configuration conf;
  private static FileSystem fs;
  private static final int BLOCK_SIZE = 1024;
  private static final int MAX_BLOCKS = 10;
  private static final int MAX_FILE_SIZE = MAX_BLOCKS * BLOCK_SIZE;
  private static final Random random = new Random();
  private static final Log LOG = LogFactory
      .getLog(TestFileStatusExtended.class);

  @Before
  public void setUpBeforeClass() throws Exception {
    conf = new Configuration();
    conf.setInt("dfs.block.size", BLOCK_SIZE);
    cluster = new MiniDFSCluster(conf, 3, true, null);
    fs = cluster.getFileSystem();
  }

  @After
  public void tearDownAfterClass() throws Exception {
    fs.close();
    cluster.shutdown();
  }

  @Test
  public void testRandomFiles() throws Exception {
    String topDir = "testRandomFiles";
    DFSTestUtil util = new DFSTestUtil(topDir, 100, 10, MAX_FILE_SIZE);
    util.createFiles(fs, topDir);
    NameNode nn = cluster.getNameNode();
    List<FileStatusExtended> stats = nn.getRandomFilesSample(0.4);
    assertNotNull(stats);
    assertTrue(stats.size() >= 10);
    for (FileStatusExtended stat : stats) {
      fs.exists(stat.getPath());
      assertEquals("", stat.getHolder());

      // Verify blocks
      Block[] statBlocks = stat.getBlocks();
      List<LocatedBlock> blocks = nn.getBlockLocations(
          stat.getPath().toString(), 0, Long.MAX_VALUE).getLocatedBlocks();
      assertEquals(blocks.size(), statBlocks.length);
      for (int i = 0; i < statBlocks.length; i++) {
        assertEquals(blocks.get(i).getBlock(), statBlocks[i]);
      }
      FileStatus st = nn.getFileInfo(stat.getPath().toString());
      assertEquals(st, (FileStatus) stat);
    }
  }

  private List<FileStatusExtended> dumpRandomFileInfo(String fileName,
      double percentage, NameNode nn) throws IOException {
    List<FileStatusExtended> stats = nn.getRandomFilesSample(percentage);
    DataOutputStream out = new DataOutputStream(new FileOutputStream(fileName));
    try {
      // Write the number of entries.
      out.writeInt(stats.size());
      for (FileStatusExtended stat : stats) {
        stat.write(out);
      }
    } finally {
      out.close();
    }
    return stats;
  }

  @Test
  public void testDump() throws Exception {
    String topDir = "testDump";
    DFSTestUtil util = new DFSTestUtil(topDir, 50, 10, MAX_FILE_SIZE);
    util.createFiles(fs, topDir);
    File fileName = new File(cluster.getBaseDataDir(), "dumpMeta");
    NameNode nn = cluster.getNameNode();
    List<FileStatusExtended> stats = dumpRandomFileInfo(
        fileName.getAbsolutePath(), 0.4, nn);
    DataInputStream in = new DataInputStream(new FileInputStream(
        fileName.getAbsolutePath()));
    int nFiles = in.readInt();
    assertEquals(stats.size(), nFiles);
    for (int i = 0; i < nFiles; i++) {
      FileStatusExtended stat = new FileStatusExtended();
      stat.readFields(in);
      assertEquals(stats.get(i), stat);
    }
  }

  @Test
  public void testPercentBasedSample() throws Exception {
    String topDir = "testPercentBasedSample";
    DFSTestUtil util = new DFSTestUtil(topDir, 200, 10, MAX_FILE_SIZE);
    util.createFiles(fs, topDir);
    NameNode nn = cluster.getNameNode();
    List<FileStatusExtended> stats = nn.getRandomFilesSample(0.1);
    assertTrue(stats.size() >= 5);
  }

  @Test
  public void testPercentBasedSampleFull() throws Exception {
    String topDir = "testPercentBasedSample";
    DFSTestUtil util = new DFSTestUtil(topDir, 15, 10, MAX_FILE_SIZE);
    util.createFiles(fs, topDir);
    NameNode nn = cluster.getNameNode();
    List<FileStatusExtended> stats = nn.getRandomFilesSample(1.0);
    assertTrue(15 >= stats.size());
  }

  @Test
  public void testInvalidPercentBasedSample1() throws Exception {
    try {
      cluster.getNameNode().getRandomFilesSample(0.0);
      fail("Did not throw : " + IllegalArgumentException.class);
    } catch (IllegalArgumentException e) {
      LOG.info("Expected exception : ", e);
    }
  }

  @Test
  public void testInvalidPercentBasedSample2() throws Exception {
    try {
      cluster.getNameNode().getRandomFilesSample(1.1);
      fail("Did not throw : " + IllegalArgumentException.class);
    } catch (IllegalArgumentException e) {
      LOG.info("Expected exception : ", e);
    }
  }

  @Test
  public void testInvalidPercentBasedSample3() throws Exception {
    try {
      cluster.getNameNode().getRandomFilesSample(-0.5);
      fail("Did not throw : " + IllegalArgumentException.class);
    } catch (IllegalArgumentException e) {
      LOG.info("Expected exception : ", e);
    }
  }

  @Test
  public void testFileUnderConstruction() throws Exception {
    String fileName = "/testFileUnderConstruction";
    FSDataOutputStream out = fs.create(new Path(fileName));
    byte [] buffer = new byte[BLOCK_SIZE * 5 + 256];
    random.nextBytes(buffer);
    out.write(buffer);
    out.sync();

    NameNode nn = cluster.getNameNode();
    List<FileStatusExtended> stats = nn.getRandomFilesSample(1);
    assertEquals(1, stats.size());
    assertEquals(((DistributedFileSystem) fs).getClient().clientName, stats
        .get(0).getHolder());
  }
}