/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.util;
import java.io.File;
import java.io.IOException;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
import org.junit.Test;
/**
* A test for MRAsyncDiskService.
*/
public class TestMRAsyncDiskService extends TestCase {
public static final Log LOG = LogFactory.getLog(TestMRAsyncDiskService.class);
private static String TEST_ROOT_DIR = new Path(System.getProperty(
"test.build.data", "/tmp")).toString();
@Override
protected void setUp() {
FileUtil.fullyDelete(new File(TEST_ROOT_DIR));
}
/**
* Given 'pathname', compute an equivalent path relative to the cwd.
* @param pathname the path to a directory.
* @return the path to that same directory, relative to ${user.dir}.
*/
private String relativeToWorking(String pathname) {
String cwd = System.getProperty("user.dir", "/");
// normalize pathname and cwd into full directory paths.
pathname = (new Path(pathname)).toUri().getPath();
cwd = (new Path(cwd)).toUri().getPath();
String [] cwdParts = cwd.split(File.separator);
String [] pathParts = pathname.split(File.separator);
// There are three possible cases:
// 1) pathname and cwd are equal. Return '.'
// 2) pathname is under cwd. Return the components that are under it.
// e.g., cwd = /a/b, path = /a/b/c, return 'c'
// 3) pathname is outside of cwd. Find the common components, if any,
// and subtract them from the returned path, then return enough '..'
// components to "undo" the non-common components of cwd, then all
// the remaining parts of pathname.
// e.g., cwd = /a/b, path = /a/c, return '../c'
if (cwd.equals(pathname)) {
LOG.info("relative to working: " + pathname + " -> .");
return "."; // They match exactly.
}
// Determine how many path components are in common between cwd and path.
int common = 0;
for (int i = 0; i < Math.min(cwdParts.length, pathParts.length); i++) {
if (cwdParts[i].equals(pathParts[i])) {
common++;
} else {
break;
}
}
// output path stringbuilder.
StringBuilder sb = new StringBuilder();
// For everything in cwd that isn't in pathname, add a '..' to undo it.
int parentDirsRequired = cwdParts.length - common;
for (int i = 0; i < parentDirsRequired; i++) {
sb.append("..");
sb.append(File.separator);
}
// Then append all non-common parts of 'pathname' itself.
for (int i = common; i < pathParts.length; i++) {
sb.append(pathParts[i]);
sb.append(File.separator);
}
// Don't end with a '/'.
String s = sb.toString();
if (s.endsWith(File.separator)) {
s = s.substring(0, s.length() - 1);
}
LOG.info("relative to working: " + pathname + " -> " + s);
return s;
}
@Test
/** Test that the relativeToWorking() method above does what we expect. */
public void testRelativeToWorking() {
assertEquals(".", relativeToWorking(System.getProperty("user.dir", ".")));
String cwd = System.getProperty("user.dir", ".");
Path cwdPath = new Path(cwd);
Path subdir = new Path(cwdPath, "foo");
assertEquals("foo", relativeToWorking(subdir.toUri().getPath()));
Path subsubdir = new Path(subdir, "bar");
assertEquals("foo/bar", relativeToWorking(subsubdir.toUri().getPath()));
Path parent = new Path(cwdPath, "..");
assertEquals("..", relativeToWorking(parent.toUri().getPath()));
Path sideways = new Path(parent, "baz");
assertEquals("../baz", relativeToWorking(sideways.toUri().getPath()));
}
@Test
/** Test that volumes specified as relative paths are handled properly
* by MRAsyncDiskService (MAPREDUCE-1887).
*/
public void testVolumeNormalization() throws Throwable {
LOG.info("TEST_ROOT_DIR is " + TEST_ROOT_DIR);
String relativeTestRoot = relativeToWorking(TEST_ROOT_DIR);
FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
String [] vols = new String[] { relativeTestRoot + "/0",
relativeTestRoot + "/1" };
// Put a file in one of the volumes to be cleared on startup.
Path delDir = new Path(vols[0], MRAsyncDiskService.TOBEDELETED);
localFileSystem.mkdirs(delDir);
localFileSystem.create(new Path(delDir, "foo")).close();
MRAsyncDiskService service = new MRAsyncDiskService(
localFileSystem, vols);
makeSureCleanedUp(vols, service);
}
/**
* This test creates some directories and then removes them through
* MRAsyncDiskService.
*/
@Test
public void testMRAsyncDiskService() throws Throwable {
FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
String[] vols = new String[]{TEST_ROOT_DIR + "/0",
TEST_ROOT_DIR + "/1"};
MRAsyncDiskService service = new MRAsyncDiskService(
localFileSystem, vols);
String a = "a";
String b = "b";
String c = "b/c";
String d = "d";
File fa = new File(vols[0], a);
File fb = new File(vols[1], b);
File fc = new File(vols[1], c);
File fd = new File(vols[1], d);
// Create the directories
fa.mkdirs();
fb.mkdirs();
fc.mkdirs();
fd.mkdirs();
assertTrue(fa.exists());
assertTrue(fb.exists());
assertTrue(fc.exists());
assertTrue(fd.exists());
// Move and delete them
service.moveAndDeleteRelativePath(vols[0], a);
assertFalse(fa.exists());
service.moveAndDeleteRelativePath(vols[1], b);
assertFalse(fb.exists());
assertFalse(fc.exists());
assertFalse(service.moveAndDeleteRelativePath(vols[1], "not_exists"));
// asyncDiskService is NOT able to delete files outside all volumes.
IOException ee = null;
try {
service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2");
} catch (IOException e) {
ee = e;
}
assertNotNull("asyncDiskService should not be able to delete files "
+ "outside all volumes", ee);
// asyncDiskService is able to automatically find the file in one
// of the volumes.
assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d));
// Make sure everything is cleaned up
makeSureCleanedUp(vols, service);
}
/**
* This test creates some directories inside the volume roots, and then
* call asyncDiskService.MoveAndDeleteAllVolumes.
* We should be able to delete all files/dirs inside the volumes except
* the toBeDeleted directory.
*/
@Test
public void testMRAsyncDiskServiceMoveAndDeleteAllVolumes() throws Throwable {
FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
String[] vols = new String[]{TEST_ROOT_DIR + "/0",
TEST_ROOT_DIR + "/1"};
MRAsyncDiskService service = new MRAsyncDiskService(
localFileSystem, vols);
String a = "a";
String b = "b";
String c = "b/c";
String d = "d";
File fa = new File(vols[0], a);
File fb = new File(vols[1], b);
File fc = new File(vols[1], c);
File fd = new File(vols[1], d);
// Create the directories
fa.mkdirs();
fb.mkdirs();
fc.mkdirs();
fd.mkdirs();
assertTrue(fa.exists());
assertTrue(fb.exists());
assertTrue(fc.exists());
assertTrue(fd.exists());
// Delete all of them
service.cleanupAllVolumes();
assertFalse(fa.exists());
assertFalse(fb.exists());
assertFalse(fc.exists());
assertFalse(fd.exists());
// Make sure everything is cleaned up
makeSureCleanedUp(vols, service);
}
/**
* This test creates some directories inside the toBeDeleted directory and
* then start the asyncDiskService.
* AsyncDiskService will create tasks to delete the content inside the
* toBeDeleted directories.
*/
@Test
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
String[] vols = new String[]{TEST_ROOT_DIR + "/0",
TEST_ROOT_DIR + "/1"};
String a = "a";
String b = "b";
String c = "b/c";
String d = "d";
// Create directories inside SUBDIR
String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED;
File fa = new File(vols[0] + suffix, a);
File fb = new File(vols[1] + suffix, b);
File fc = new File(vols[1] + suffix, c);
File fd = new File(vols[1] + suffix, d);
// Create the directories
fa.mkdirs();
fb.mkdirs();
fc.mkdirs();
fd.mkdirs();
assertTrue(fa.exists());
assertTrue(fb.exists());
assertTrue(fc.exists());
assertTrue(fd.exists());
// Create the asyncDiskService which will delete all contents inside SUBDIR
MRAsyncDiskService service = new MRAsyncDiskService(
localFileSystem, vols);
// Make sure everything is cleaned up
makeSureCleanedUp(vols, service);
}
private void makeSureCleanedUp(String[] vols, MRAsyncDiskService service)
throws Throwable {
// Sleep at most 5 seconds to make sure the deleted items are all gone.
service.shutdown();
if (!service.awaitTermination(5000)) {
fail("MRAsyncDiskService is still not shutdown in 5 seconds!");
}
// All contents should be gone by now.
for (int i = 0; i < vols.length; i++) {
File subDir = new File(vols[0]);
String[] subDirContent = subDir.list();
assertEquals("Volume should contain a single child: "
+ MRAsyncDiskService.TOBEDELETED, 1, subDirContent.length);
File toBeDeletedDir = new File(vols[0], MRAsyncDiskService.TOBEDELETED);
String[] content = toBeDeletedDir.list();
assertNotNull("Cannot find " + toBeDeletedDir, content);
assertEquals("" + toBeDeletedDir + " should be empty now.", 0,
content.length);
}
}
@Test
public void testToleratesSomeUnwritableVolumes() throws Throwable {
FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
String[] vols = new String[]{TEST_ROOT_DIR + "/0",
TEST_ROOT_DIR + "/1"};
assertTrue(new File(vols[0]).mkdirs());
assertEquals(0, FileUtil.chmod(vols[0], "400")); // read only
try {
new MRAsyncDiskService(localFileSystem, vols);
} finally {
FileUtil.chmod(vols[0], "755"); // make writable again
}
}
}