/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
import junit.framework.TestCase;
import java.net.InetSocketAddress;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil.ErrorSimulator;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.CheckpointStorage;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.DelayAnswer;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Level;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import com.google.common.base.Joiner;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.primitives.Ints;
import static org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil.assertNNHasCheckpoints;
import static org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil.getNameNodeCurrentDirs;
/**
* This class tests the creation and validation of a checkpoint.
*/
public class TestCheckpoint extends TestCase {
static {
((Log4JLogger)FSImage.LOG).getLogger().setLevel(Level.ALL);
}
static final Log LOG = LogFactory.getLog(TestCheckpoint.class);
static final long seed = 0xDEADBEEFL;
static final int blockSize = 4096;
static final int fileSize = 8192;
static final int numDatanodes = 3;
short replication = 3;
@Override
public void setUp() throws IOException {
FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory()));
ErrorSimulator.initializeErrorSimulationEvent(5);
}
static void writeFile(FileSystem fileSys, Path name, int repl)
throws IOException {
FSDataOutputStream stm = fileSys.create(name, true,
fileSys.getConf().getInt("io.file.buffer.size", 4096),
(short)repl, (long)blockSize);
byte[] buffer = new byte[TestCheckpoint.fileSize];
Random rand = new Random(TestCheckpoint.seed);
rand.nextBytes(buffer);
stm.write(buffer);
stm.close();
}
static void checkFile(FileSystem fileSys, Path name, int repl)
throws IOException {
assertTrue(fileSys.exists(name));
int replication = fileSys.getFileStatus(name).getReplication();
assertEquals("replication for " + name, repl, replication);
//We should probably test for more of the file properties.
}
static void cleanupFile(FileSystem fileSys, Path name)
throws IOException {
assertTrue(fileSys.exists(name));
fileSys.delete(name, true);
assertTrue(!fileSys.exists(name));
}
/*
* Verify that namenode does not startup if one namedir is bad.
*/
public void testNameDirError() throws IOException {
LOG.info("Starting testNameDirError");
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.build();
Collection<URI> nameDirs = cluster.getNameDirs(0);
cluster.shutdown();
cluster = null;
for (URI nameDirUri : nameDirs) {
File dir = new File(nameDirUri.getPath());
try {
// Simulate the mount going read-only
dir.setWritable(false);
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.format(false)
.build();
fail("NN should have failed to start with " + dir + " set unreadable");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
"storage directory does not exist or is not accessible",
ioe);
} finally {
if (cluster != null) {
cluster.shutdown();
cluster = null;
}
dir.setWritable(true);
}
}
}
/**
* Checks that an IOException in NNStorage.writeTransactionIdFile is handled
* correctly (by removing the storage directory)
* See https://issues.apache.org/jira/browse/HDFS-2011
*/
public void testWriteTransactionIdHandlesIOE() throws Exception {
LOG.info("Check IOException handled correctly by writeTransactionIdFile");
ArrayList<URI> fsImageDirs = new ArrayList<URI>();
ArrayList<URI> editsDirs = new ArrayList<URI>();
File filePath =
new File(System.getProperty("test.build.data","/tmp"), "storageDirToCheck");
assertTrue("Couldn't create directory storageDirToCheck",
filePath.exists() || filePath.mkdirs());
fsImageDirs.add(filePath.toURI());
editsDirs.add(filePath.toURI());
NNStorage nnStorage = new NNStorage(new HdfsConfiguration(),
fsImageDirs, editsDirs);
try {
assertTrue("List of storage directories didn't have storageDirToCheck.",
nnStorage.getEditsDirectories().iterator().next().
toString().indexOf("storageDirToCheck") != -1);
assertTrue("List of removed storage directories wasn't empty",
nnStorage.getRemovedStorageDirs().isEmpty());
} finally {
// Delete storage directory to cause IOException in writeTransactionIdFile
assertTrue("Couldn't remove directory " + filePath.getAbsolutePath(),
filePath.delete());
}
// Just call writeTransactionIdFile using any random number
nnStorage.writeTransactionIdFileToStorage(1);
List<StorageDirectory> listRsd = nnStorage.getRemovedStorageDirs();
assertTrue("Removed directory wasn't what was expected",
listRsd.size() > 0 && listRsd.get(listRsd.size() - 1).getRoot().
toString().indexOf("storageDirToCheck") != -1);
}
/*
* Simulate namenode crashing after rolling edit log.
*/
public void testSecondaryNamenodeError1()
throws IOException {
LOG.info("Starting testSecondaryNamenodeError1");
Configuration conf = new HdfsConfiguration();
Path file1 = new Path("checkpointxx.dat");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes)
.build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
try {
assertTrue(!fileSys.exists(file1));
//
// Make the checkpoint fail after rolling the edits log.
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
ErrorSimulator.setErrorSimulation(0);
try {
secondary.doCheckpoint(); // this should fail
assertTrue(false);
} catch (IOException e) {
}
ErrorSimulator.clearErrorSimulation(0);
secondary.shutdown();
//
// Create a new file
//
writeFile(fileSys, file1, replication);
checkFile(fileSys, file1, replication);
} finally {
fileSys.close();
cluster.shutdown();
}
//
// Restart cluster and verify that file exists.
// Then take another checkpoint to verify that the
// namenode restart accounted for the rolled edit logs.
//
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
.format(false).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
try {
checkFile(fileSys, file1, replication);
cleanupFile(fileSys, file1);
SecondaryNameNode secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint();
secondary.shutdown();
} finally {
fileSys.close();
cluster.shutdown();
}
}
/*
* Simulate a namenode crash after uploading new image
*/
public void testSecondaryNamenodeError2() throws IOException {
LOG.info("Starting testSecondaryNamenodeError2");
Configuration conf = new HdfsConfiguration();
Path file1 = new Path("checkpointyy.dat");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes)
.build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
try {
assertTrue(!fileSys.exists(file1));
//
// Make the checkpoint fail after uploading the new fsimage.
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
ErrorSimulator.setErrorSimulation(1);
try {
secondary.doCheckpoint(); // this should fail
assertTrue(false);
} catch (IOException e) {
}
ErrorSimulator.clearErrorSimulation(1);
secondary.shutdown();
//
// Create a new file
//
writeFile(fileSys, file1, replication);
checkFile(fileSys, file1, replication);
} finally {
fileSys.close();
cluster.shutdown();
}
//
// Restart cluster and verify that file exists.
// Then take another checkpoint to verify that the
// namenode restart accounted for the rolled edit logs.
//
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
try {
checkFile(fileSys, file1, replication);
cleanupFile(fileSys, file1);
SecondaryNameNode secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint();
secondary.shutdown();
} finally {
fileSys.close();
cluster.shutdown();
}
}
/*
* Simulate a secondary namenode crash after rolling the edit log.
*/
public void testSecondaryNamenodeError3() throws IOException {
LOG.info("Starting testSecondaryNamenodeError3");
Configuration conf = new HdfsConfiguration();
Path file1 = new Path("checkpointzz.dat");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes)
.build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
try {
assertTrue(!fileSys.exists(file1));
//
// Make the checkpoint fail after rolling the edit log.
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
ErrorSimulator.setErrorSimulation(0);
try {
secondary.doCheckpoint(); // this should fail
assertTrue(false);
} catch (IOException e) {
}
ErrorSimulator.clearErrorSimulation(0);
secondary.shutdown(); // secondary namenode crash!
// start new instance of secondary and verify that
// a new rollEditLog suceedes inspite of the fact that
// edits.new already exists.
//
secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint(); // this should work correctly
secondary.shutdown();
//
// Create a new file
//
writeFile(fileSys, file1, replication);
checkFile(fileSys, file1, replication);
} finally {
fileSys.close();
cluster.shutdown();
}
//
// Restart cluster and verify that file exists.
// Then take another checkpoint to verify that the
// namenode restart accounted for the twice-rolled edit logs.
//
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
try {
checkFile(fileSys, file1, replication);
cleanupFile(fileSys, file1);
SecondaryNameNode secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint();
secondary.shutdown();
} finally {
fileSys.close();
cluster.shutdown();
}
}
/**
* Simulate a secondary node failure to transfer image
* back to the name-node.
* Used to truncate primary fsimage file.
*/
public void testSecondaryFailsToReturnImage() throws IOException {
LOG.info("Starting testSecondaryFailsToReturnImage");
Configuration conf = new HdfsConfiguration();
Path file1 = new Path("checkpointRI.dat");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes)
.build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
FSImage image = cluster.getNameNode().getFSImage();
try {
assertTrue(!fileSys.exists(file1));
StorageDirectory sd = image.getStorage().getStorageDir(0);
File latestImageBeforeCheckpoint = FSImageTestUtil.findLatestImageFile(sd);
long fsimageLength = latestImageBeforeCheckpoint.length();
//
// Make the checkpoint
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
ErrorSimulator.setErrorSimulation(2);
try {
secondary.doCheckpoint(); // this should fail
fail("Checkpoint succeeded even though we injected an error!");
} catch (IOException e) {
// check that it's the injected exception
GenericTestUtils.assertExceptionContains(
"If this exception is not caught", e);
}
ErrorSimulator.clearErrorSimulation(2);
// Verify that image file sizes did not change.
for (StorageDirectory sd2 :
image.getStorage().dirIterable(NameNodeDirType.IMAGE)) {
File thisNewestImage = FSImageTestUtil.findLatestImageFile(sd2);
long len = thisNewestImage.length();
assertEquals(fsimageLength, len);
}
secondary.shutdown();
} finally {
fileSys.close();
cluster.shutdown();
}
}
/**
* Simulate 2NN failing to send the whole file (error type 3)
* The length header in the HTTP transfer should prevent
* this from corrupting the NN.
*/
public void testNameNodeImageSendFailWrongSize()
throws IOException {
LOG.info("Starting testNameNodeImageSendFailWrongSize");
doSendFailTest(3, "is not of the advertised size");
}
/**
* Simulate 2NN sending a corrupt image (error type 4)
* The digest header in the HTTP transfer should prevent
* this from corrupting the NN.
*/
public void testNameNodeImageSendFailWrongDigest()
throws IOException {
LOG.info("Starting testNameNodeImageSendFailWrongDigest");
doSendFailTest(4, "does not match advertised digest");
}
/**
* Run a test where the 2NN runs into some kind of error when
* sending the checkpoint back to the NN.
* @param errorType the ErrorSimulator type to trigger
* @param exceptionSubstring an expected substring of the triggered exception
*/
private void doSendFailTest(int errorType, String exceptionSubstring)
throws IOException {
Configuration conf = new HdfsConfiguration();
Path file1 = new Path("checkpoint-doSendFailTest-" + errorType + ".dat");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes)
.build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
try {
assertTrue(!fileSys.exists(file1));
//
// Make the checkpoint fail after rolling the edit log.
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
ErrorSimulator.setErrorSimulation(errorType);
try {
secondary.doCheckpoint(); // this should fail
fail("Did not get expected exception");
} catch (IOException e) {
// We only sent part of the image. Have to trigger this exception
GenericTestUtils.assertExceptionContains(exceptionSubstring, e);
}
ErrorSimulator.clearErrorSimulation(errorType);
secondary.shutdown(); // secondary namenode crash!
// start new instance of secondary and verify that
// a new rollEditLog succedes in spite of the fact that we had
// a partially failed checkpoint previously.
//
secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint(); // this should work correctly
secondary.shutdown();
//
// Create a new file
//
writeFile(fileSys, file1, replication);
checkFile(fileSys, file1, replication);
} finally {
fileSys.close();
cluster.shutdown();
}
}
/**
* Test that the NN locks its storage and edits directories, and won't start up
* if the directories are already locked
**/
public void testNameDirLocking() throws IOException {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.build();
// Start a NN, and verify that lock() fails in all of the configured
// directories
StorageDirectory savedSd = null;
try {
NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
for (StorageDirectory sd : storage.dirIterable(null)) {
assertLockFails(sd);
savedSd = sd;
}
} finally {
cluster.shutdown();
}
assertNotNull(savedSd);
// Lock one of the saved directories, then start the NN, and make sure it
// fails to start
assertClusterStartFailsWhenDirLocked(conf, savedSd);
}
/**
* Test that, if the edits dir is separate from the name dir, it is
* properly locked.
**/
public void testSeparateEditsDirLocking() throws IOException {
Configuration conf = new HdfsConfiguration();
File editsDir = new File(MiniDFSCluster.getBaseDirectory() +
"/testSeparateEditsDirLocking");
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
editsDir.getAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.manageNameDfsDirs(false)
.numDataNodes(0)
.build();
// Start a NN, and verify that lock() fails in all of the configured
// directories
StorageDirectory savedSd = null;
try {
NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
assertEquals(editsDir.getAbsoluteFile(), sd.getRoot());
assertLockFails(sd);
savedSd = sd;
}
} finally {
cluster.shutdown();
}
assertNotNull(savedSd);
// Lock one of the saved directories, then start the NN, and make sure it
// fails to start
assertClusterStartFailsWhenDirLocked(conf, savedSd);
}
/**
* Test that the SecondaryNameNode properly locks its storage directories.
*/
public void testSecondaryNameNodeLocking() throws Exception {
// Start a primary NN so that the secondary will start successfully
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.build();
SecondaryNameNode secondary = null;
try {
StorageDirectory savedSd = null;
// Start a secondary NN, then make sure that all of its storage
// dirs got locked.
secondary = startSecondaryNameNode(conf);
NNStorage storage = secondary.getFSImage().getStorage();
for (StorageDirectory sd : storage.dirIterable(null)) {
assertLockFails(sd);
savedSd = sd;
}
LOG.info("===> Shutting down first 2NN");
secondary.shutdown();
secondary = null;
LOG.info("===> Locking a dir, starting second 2NN");
// Lock one of its dirs, make sure it fails to start
LOG.info("Trying to lock" + savedSd);
savedSd.lock();
try {
secondary = startSecondaryNameNode(conf);
assertFalse("Should fail to start 2NN when " + savedSd + " is locked",
savedSd.isLockSupported());
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains("already locked", ioe);
} finally {
savedSd.unlock();
}
} finally {
if (secondary != null) {
secondary.shutdown();
}
cluster.shutdown();
}
}
/**
* Assert that the given storage directory can't be locked, because
* it's already locked.
*/
private static void assertLockFails(StorageDirectory sd) {
try {
sd.lock();
// If the above line didn't throw an exception, then
// locking must not be supported
assertFalse(sd.isLockSupported());
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains("already locked", ioe);
}
}
/**
* Assert that, if sdToLock is locked, the cluster is not allowed to start up.
* @param conf cluster conf to use
* @param sdToLock the storage directory to lock
*/
private static void assertClusterStartFailsWhenDirLocked(
Configuration conf, StorageDirectory sdToLock) throws IOException {
// Lock the edits dir, then start the NN, and make sure it fails to start
sdToLock.lock();
try {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.manageNameDfsDirs(false)
.numDataNodes(0)
.build();
assertFalse("cluster should fail to start after locking " +
sdToLock, sdToLock.isLockSupported());
cluster.shutdown();
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains("already locked", ioe);
} finally {
sdToLock.unlock();
}
}
/**
* Test the importCheckpoint startup option. Verifies:
* 1. if the NN already contains an image, it will not be allowed
* to import a checkpoint.
* 2. if the NN does not contain an image, importing a checkpoint
* succeeds and re-saves the image
*/
public void testImportCheckpoint() throws Exception {
Configuration conf = new HdfsConfiguration();
Path testPath = new Path("/testfile");
SecondaryNameNode snn = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.build();
Collection<URI> nameDirs = cluster.getNameDirs(0);
try {
// Make an entry in the namespace, used for verifying checkpoint
// later.
cluster.getFileSystem().mkdirs(testPath);
// Take a checkpoint
snn = startSecondaryNameNode(conf);
snn.doCheckpoint();
} finally {
if (snn != null) {
snn.shutdown();
}
cluster.shutdown();
cluster = null;
}
LOG.info("Trying to import checkpoint when the NameNode already " +
"contains an image. This should fail.");
try {
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.format(false)
.startupOption(StartupOption.IMPORT)
.build();
fail("NameNode did not fail to start when it already contained " +
"an image");
} catch (IOException ioe) {
// Expected
GenericTestUtils.assertExceptionContains(
"NameNode already contains an image", ioe);
} finally {
if (cluster != null) {
cluster.shutdown();
cluster = null;
}
}
LOG.info("Removing NN storage contents");
for(URI uri : nameDirs) {
File dir = new File(uri.getPath());
LOG.info("Cleaning " + dir);
removeAndRecreateDir(dir);
}
LOG.info("Trying to import checkpoint");
try {
cluster = new MiniDFSCluster.Builder(conf)
.format(false)
.numDataNodes(0)
.startupOption(StartupOption.IMPORT)
.build();
assertTrue("Path from checkpoint should exist after import",
cluster.getFileSystem().exists(testPath));
// Make sure that the image got saved on import
FSImageTestUtil.assertNNHasCheckpoints(cluster, Ints.asList(3));
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
private static void removeAndRecreateDir(File dir) throws IOException {
if(dir.exists())
if(!(FileUtil.fullyDelete(dir)))
throw new IOException("Cannot remove directory: " + dir);
if (!dir.mkdirs())
throw new IOException("Cannot create directory " + dir);
}
SecondaryNameNode startSecondaryNameNode(Configuration conf
) throws IOException {
conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0");
return new SecondaryNameNode(conf);
}
SecondaryNameNode startSecondaryNameNode(Configuration conf, int index)
throws IOException {
Configuration snnConf = new Configuration(conf);
snnConf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0");
snnConf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY,
MiniDFSCluster.getBaseDirectory() + "/2nn-" + index);
return new SecondaryNameNode(snnConf);
}
/**
* Tests checkpoint in HDFS.
*/
public void testCheckpoint() throws IOException {
Path file1 = new Path("checkpoint.dat");
Path file2 = new Path("checkpoint2.dat");
Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0");
replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes).build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
try {
//
// verify that 'format' really blew away all pre-existing files
//
assertTrue(!fileSys.exists(file1));
assertTrue(!fileSys.exists(file2));
//
// Create file1
//
writeFile(fileSys, file1, replication);
checkFile(fileSys, file1, replication);
//
// Take a checkpoint
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint();
secondary.shutdown();
} finally {
fileSys.close();
cluster.shutdown();
}
//
// Restart cluster and verify that file1 still exist.
//
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
Path tmpDir = new Path("/tmp_tmp");
try {
// check that file1 still exists
checkFile(fileSys, file1, replication);
cleanupFile(fileSys, file1);
// create new file file2
writeFile(fileSys, file2, replication);
checkFile(fileSys, file2, replication);
//
// Take a checkpoint
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
secondary.doCheckpoint();
fileSys.delete(tmpDir, true);
fileSys.mkdirs(tmpDir);
secondary.doCheckpoint();
secondary.shutdown();
} finally {
fileSys.close();
cluster.shutdown();
}
//
// Restart cluster and verify that file2 exists and
// file1 does not exist.
//
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
assertTrue(!fileSys.exists(file1));
assertTrue(fileSys.exists(tmpDir));
try {
// verify that file2 exists
checkFile(fileSys, file2, replication);
} finally {
fileSys.close();
cluster.shutdown();
}
}
/**
* Tests save namepsace.
*/
public void testSaveNamespace() throws IOException {
MiniDFSCluster cluster = null;
DistributedFileSystem fs = null;
FileContext fc;
try {
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(true).build();
cluster.waitActive();
fs = (DistributedFileSystem)(cluster.getFileSystem());
fc = FileContext.getFileContext(cluster.getURI(0));
// Saving image without safe mode should fail
DFSAdmin admin = new DFSAdmin(conf);
String[] args = new String[]{"-saveNamespace"};
try {
admin.run(args);
} catch(IOException eIO) {
assertTrue(eIO.getLocalizedMessage().contains("Safe mode should be turned ON"));
} catch(Exception e) {
throw new IOException(e);
}
// create new file
Path file = new Path("namespace.dat");
writeFile(fs, file, replication);
checkFile(fs, file, replication);
// create new link
Path symlink = new Path("file.link");
fc.createSymlink(file, symlink, false);
assertTrue(fc.getFileLinkStatus(symlink).isSymlink());
// verify that the edits file is NOT empty
Collection<URI> editsDirs = cluster.getNameEditsDirs(0);
for(URI uri : editsDirs) {
File ed = new File(uri.getPath());
assertTrue(new File(ed, "current/"
+ NNStorage.getInProgressEditsFileName(1))
.length() > Integer.SIZE/Byte.SIZE);
}
// Saving image in safe mode should succeed
fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
try {
admin.run(args);
} catch(Exception e) {
throw new IOException(e);
}
// the following steps should have happened:
// edits_inprogress_1 -> edits_1-8 (finalized)
// fsimage_8 created
// edits_inprogress_9 created
//
for(URI uri : editsDirs) {
File ed = new File(uri.getPath());
File curDir = new File(ed, "current");
LOG.info("Files in " + curDir + ":\n " +
Joiner.on("\n ").join(curDir.list()));
// Verify that the first edits file got finalized
File originalEdits = new File(curDir,
NNStorage.getInProgressEditsFileName(1));
assertFalse(originalEdits.exists());
File finalizedEdits = new File(curDir,
NNStorage.getFinalizedEditsFileName(1,8));
assertTrue(finalizedEdits.exists());
assertTrue(finalizedEdits.length() > Integer.SIZE/Byte.SIZE);
assertTrue(new File(ed, "current/"
+ NNStorage.getInProgressEditsFileName(9)).exists());
}
Collection<URI> imageDirs = cluster.getNameDirs(0);
for (URI uri : imageDirs) {
File imageDir = new File(uri.getPath());
File savedImage = new File(imageDir, "current/"
+ NNStorage.getImageFileName(8));
assertTrue("Should have saved image at " + savedImage,
savedImage.exists());
}
// restart cluster and verify file exists
cluster.shutdown();
cluster = null;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
cluster.waitActive();
fs = (DistributedFileSystem)(cluster.getFileSystem());
checkFile(fs, file, replication);
fc = FileContext.getFileContext(cluster.getURI(0));
assertTrue(fc.getFileLinkStatus(symlink).isSymlink());
} finally {
try {
if(fs != null) fs.close();
if(cluster!= null) cluster.shutdown();
} catch (Throwable t) {
LOG.error("Failed to shutdown", t);
}
}
}
/* Test case to test CheckpointSignature */
@SuppressWarnings("deprecation")
public void testCheckpointSignature() throws IOException {
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
.format(true).build();
NameNode nn = cluster.getNameNode();
NamenodeProtocols nnRpc = nn.getRpcServer();
SecondaryNameNode secondary = startSecondaryNameNode(conf);
// prepare checkpoint image
secondary.doCheckpoint();
CheckpointSignature sig = nnRpc.rollEditLog();
// manipulate the CheckpointSignature fields
sig.setBlockpoolID("somerandomebpid");
sig.clusterID = "somerandomcid";
try {
sig.validateStorageInfo(nn.getFSImage()); // this should fail
assertTrue("This test is expected to fail.", false);
} catch (Exception ignored) {
}
secondary.shutdown();
cluster.shutdown();
}
/**
* Tests the following sequence of events:
* - secondary successfully makes a checkpoint
* - it then fails while trying to upload it
* - it then fails again for the same reason
* - it then tries to checkpoint a third time
*/
public void testCheckpointAfterTwoFailedUploads() throws IOException {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
Configuration conf = new HdfsConfiguration();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
.format(true).build();
secondary = startSecondaryNameNode(conf);
ErrorSimulator.setErrorSimulation(1);
// Fail to checkpoint once
try {
secondary.doCheckpoint();
fail("Should have failed upload");
} catch (IOException ioe) {
LOG.info("Got expected failure", ioe);
assertTrue(ioe.toString().contains("Simulating error1"));
}
// Fail to checkpoint again
try {
secondary.doCheckpoint();
fail("Should have failed upload");
} catch (IOException ioe) {
LOG.info("Got expected failure", ioe);
assertTrue(ioe.toString().contains("Simulating error1"));
} finally {
ErrorSimulator.clearErrorSimulation(1);
}
// Now with the cleared error simulation, it should succeed
secondary.doCheckpoint();
} finally {
if (secondary != null) {
secondary.shutdown();
}
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Starts two namenodes and two secondary namenodes, verifies that secondary
* namenodes are configured correctly to talk to their respective namenodes
* and can do the checkpoint.
*
* @throws IOException
*/
public void testMultipleSecondaryNamenodes() throws IOException {
Configuration conf = new HdfsConfiguration();
String nameserviceId1 = "ns1";
String nameserviceId2 = "ns2";
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, nameserviceId1
+ "," + nameserviceId2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numNameNodes(2)
.nameNodePort(9928).build();
Configuration snConf1 = new HdfsConfiguration(cluster.getConfiguration(0));
Configuration snConf2 = new HdfsConfiguration(cluster.getConfiguration(1));
InetSocketAddress nn1RpcAddress =
cluster.getNameNode(0).getNameNodeAddress();
InetSocketAddress nn2RpcAddress =
cluster.getNameNode(1).getNameNodeAddress();
String nn1 = nn1RpcAddress.getHostName() + ":" + nn1RpcAddress.getPort();
String nn2 = nn2RpcAddress.getHostName() + ":" + nn2RpcAddress.getPort();
// Set the Service Rpc address to empty to make sure the node specific
// setting works
snConf1.set(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "");
snConf2.set(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "");
// Set the nameserviceIds
snConf1.set(DFSUtil.getNameServiceIdKey(
DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nameserviceId1), nn1);
snConf2.set(DFSUtil.getNameServiceIdKey(
DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nameserviceId2), nn2);
SecondaryNameNode secondary1 = startSecondaryNameNode(snConf1);
SecondaryNameNode secondary2 = startSecondaryNameNode(snConf2);
// make sure the two secondary namenodes are talking to correct namenodes.
assertEquals(secondary1.getNameNodeAddress().getPort(), nn1RpcAddress.getPort());
assertEquals(secondary2.getNameNodeAddress().getPort(), nn2RpcAddress.getPort());
assertTrue(secondary1.getNameNodeAddress().getPort() != secondary2
.getNameNodeAddress().getPort());
// both should checkpoint.
secondary1.doCheckpoint();
secondary2.doCheckpoint();
secondary1.shutdown();
secondary2.shutdown();
cluster.shutdown();
}
/**
* Test that the secondary doesn't have to re-download image
* if it hasn't changed.
*/
public void testSecondaryImageDownload() throws IOException {
LOG.info("Starting testSecondaryImageDownload");
Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0");
Path dir = new Path("/checkpoint");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDatanodes)
.format(true).build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
FSImage image = cluster.getNameNode().getFSImage();
try {
assertTrue(!fileSys.exists(dir));
//
// Make the checkpoint
//
SecondaryNameNode secondary = startSecondaryNameNode(conf);
File secondaryDir = new File(MiniDFSCluster.getBaseDirectory(), "namesecondary1");
File secondaryCurrent = new File(secondaryDir, "current");
long expectedTxIdToDownload = cluster.getNameNode().getFSImage()
.getStorage().getMostRecentCheckpointTxId();
File secondaryFsImageBefore = new File(secondaryCurrent,
NNStorage.getImageFileName(expectedTxIdToDownload));
File secondaryFsImageAfter = new File(secondaryCurrent,
NNStorage.getImageFileName(expectedTxIdToDownload + 2));
assertFalse("Secondary should start with empty current/ dir " +
"but " + secondaryFsImageBefore + " exists",
secondaryFsImageBefore.exists());
assertTrue("Secondary should have loaded an image",
secondary.doCheckpoint());
assertTrue("Secondary should have downloaded original image",
secondaryFsImageBefore.exists());
assertTrue("Secondary should have created a new image",
secondaryFsImageAfter.exists());
long fsimageLength = secondaryFsImageBefore.length();
assertEquals("Image size should not have changed",
fsimageLength,
secondaryFsImageAfter.length());
// change namespace
fileSys.mkdirs(dir);
assertFalse("Another checkpoint should not have to re-load image",
secondary.doCheckpoint());
for (StorageDirectory sd :
image.getStorage().dirIterable(NameNodeDirType.IMAGE)) {
File imageFile = NNStorage.getImageFile(sd,
expectedTxIdToDownload + 5);
assertTrue("Image size increased",
imageFile.length() > fsimageLength);
}
secondary.shutdown();
} finally {
fileSys.close();
cluster.shutdown();
}
}
/**
* Test case where two secondary namenodes are checkpointing the same
* NameNode. This differs from {@link #testMultipleSecondaryNamenodes()}
* since that test runs against two distinct NNs.
*
* This case tests the following interleaving:
* - 2NN A downloads image (up to txid 2)
* - 2NN A about to save its own checkpoint
* - 2NN B downloads image (up to txid 4)
* - 2NN B uploads checkpoint (txid 4)
* - 2NN A uploads checkpoint (txid 2)
*
* It verifies that this works even though the earlier-txid checkpoint gets
* uploaded after the later-txid checkpoint.
*/
public void testMultipleSecondaryNNsAgainstSameNN() throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.format(true).build();
SecondaryNameNode secondary1 = null, secondary2 = null;
try {
// Start 2NNs
secondary1 = startSecondaryNameNode(conf, 1);
secondary2 = startSecondaryNameNode(conf, 2);
// Make the first 2NN's checkpoint process delayable - we can pause it
// right before it saves its checkpoint image.
CheckpointStorage spyImage1 = spyOnSecondaryImage(secondary1);
DelayAnswer delayer = new DelayAnswer(LOG);
Mockito.doAnswer(delayer).when(spyImage1)
.saveFSImageInAllDirs(Mockito.anyLong());
// Set up a thread to do a checkpoint from the first 2NN
DoCheckpointThread checkpointThread = new DoCheckpointThread(secondary1);
checkpointThread.start();
// Wait for the first checkpointer to get to where it should save its image.
delayer.waitForCall();
// Now make the second checkpointer run an entire checkpoint
secondary2.doCheckpoint();
// Let the first one finish
delayer.proceed();
// It should have succeeded even though another checkpoint raced with it.
checkpointThread.join();
checkpointThread.propagateExceptions();
// primary should record "last checkpoint" as the higher txid (even though
// a checkpoint with a lower txid finished most recently)
NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
assertEquals(4, storage.getMostRecentCheckpointTxId());
// Should have accepted both checkpoints
assertNNHasCheckpoints(cluster, ImmutableList.of(2,4));
// Now have second one checkpoint one more time just to make sure that
// the NN isn't left in a broken state
secondary2.doCheckpoint();
// NN should have received new checkpoint
assertEquals(6, storage.getMostRecentCheckpointTxId());
} finally {
cleanup(secondary1);
cleanup(secondary2);
if (cluster != null) {
cluster.shutdown();
}
}
// Validate invariant that files named the same are the same.
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
// NN should have removed the checkpoint at txid 2 at this point, but has
// one at txid 6
assertNNHasCheckpoints(cluster, ImmutableList.of(4,6));
}
/**
* Test case where two secondary namenodes are checkpointing the same
* NameNode. This differs from {@link #testMultipleSecondaryNamenodes()}
* since that test runs against two distinct NNs.
*
* This case tests the following interleaving:
* - 2NN A) calls rollEdits()
* - 2NN B) calls rollEdits()
* - 2NN A) paused at getRemoteEditLogManifest()
* - 2NN B) calls getRemoteEditLogManifest() (returns up to txid 4)
* - 2NN B) uploads checkpoint fsimage_4
* - 2NN A) allowed to proceed, also returns up to txid 4
* - 2NN A) uploads checkpoint fsimage_4 as well, should fail gracefully
*
* It verifies that one of the two gets an error that it's uploading a
* duplicate checkpoint, and the other one succeeds.
*/
public void testMultipleSecondaryNNsAgainstSameNN2() throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.format(true).build();
SecondaryNameNode secondary1 = null, secondary2 = null;
try {
// Start 2NNs
secondary1 = startSecondaryNameNode(conf, 1);
secondary2 = startSecondaryNameNode(conf, 2);
// Make the first 2NN's checkpoint process delayable - we can pause it
// right before it calls getRemoteEditLogManifest.
// The method to set up a spy on an RPC protocol is a little bit involved
// since we can't spy directly on a proxy object. This sets up a mock
// which delegates all its calls to the original object, instead.
final NamenodeProtocol origNN = secondary1.getNameNode();
final Answer<Object> delegator = new GenericTestUtils.DelegateAnswer(origNN);
NamenodeProtocol spyNN = Mockito.mock(NamenodeProtocol.class, delegator);
DelayAnswer delayer = new DelayAnswer(LOG) {
protected Object passThrough(InvocationOnMock invocation) throws Throwable {
return delegator.answer(invocation);
}
};
secondary1.setNameNode(spyNN);
Mockito.doAnswer(delayer).when(spyNN)
.getEditLogManifest(Mockito.anyLong());
// Set up a thread to do a checkpoint from the first 2NN
DoCheckpointThread checkpointThread = new DoCheckpointThread(secondary1);
checkpointThread.start();
// Wait for the first checkpointer to be about to call getEditLogManifest
delayer.waitForCall();
// Now make the second checkpointer run an entire checkpoint
secondary2.doCheckpoint();
// NN should have now received fsimage_4
NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
assertEquals(4, storage.getMostRecentCheckpointTxId());
// Let the first one finish
delayer.proceed();
// Letting the first node continue should catch an exception
checkpointThread.join();
try {
checkpointThread.propagateExceptions();
fail("Didn't throw!");
} catch (Exception ioe) {
assertTrue("Unexpected exception: " +
StringUtils.stringifyException(ioe),
ioe.toString().contains("Another checkpointer already uploaded"));
LOG.info("Caught expected exception", ioe);
}
// primary should still consider fsimage_4 the latest
assertEquals(4, storage.getMostRecentCheckpointTxId());
// Now have second one checkpoint one more time just to make sure that
// the NN isn't left in a broken state
secondary2.doCheckpoint();
assertEquals(6, storage.getMostRecentCheckpointTxId());
// Should have accepted both checkpoints
assertNNHasCheckpoints(cluster, ImmutableList.of(4,6));
// Let the first one also go again on its own to make sure it can
// continue at next checkpoint
secondary1.setNameNode(origNN);
secondary1.doCheckpoint();
// NN should have received new checkpoint
assertEquals(8, storage.getMostRecentCheckpointTxId());
} finally {
cleanup(secondary1);
cleanup(secondary2);
if (cluster != null) {
cluster.shutdown();
}
}
// Validate invariant that files named the same are the same.
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
// Validate that the NN received checkpoints at expected txids
// (i.e that both checkpoints went through)
assertNNHasCheckpoints(cluster, ImmutableList.of(6,8));
}
/**
* Test case where the name node is reformatted while the secondary namenode
* is running. The secondary should shut itself down if if talks to a NN
* with the wrong namespace.
*/
public void testReformatNNBetweenCheckpoints() throws IOException {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
Configuration conf = new HdfsConfiguration();
conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
1);
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(true).build();
int origPort = cluster.getNameNodePort();
int origHttpPort = cluster.getNameNode().getHttpAddress().getPort();
secondary = startSecondaryNameNode(conf);
// secondary checkpoints once
secondary.doCheckpoint();
// we reformat primary NN
cluster.shutdown();
cluster = null;
// Brief sleep to make sure that the 2NN's IPC connection to the NN
// is dropped.
try {
Thread.sleep(100);
} catch (InterruptedException ie) {
}
// Start a new NN with the same host/port.
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.nameNodePort(origPort)
.nameNodeHttpPort(origHttpPort)
.format(true).build();
try {
secondary.doCheckpoint();
fail("Should have failed checkpoint against a different namespace");
} catch (IOException ioe) {
LOG.info("Got expected failure", ioe);
assertTrue(ioe.toString().contains("Inconsistent checkpoint"));
}
} finally {
if (secondary != null) {
secondary.shutdown();
}
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test that the primary NN will not serve any files to a 2NN who doesn't
* share its namespace ID, and also will not accept any files from one.
*/
public void testNamespaceVerifiedOnFileTransfer() throws IOException {
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(true).build();
NamenodeProtocols nn = cluster.getNameNodeRpc();
String fsName = NameNode.getHostPortString(
cluster.getNameNode().getHttpAddress());
// Make a finalized log on the server side.
nn.rollEditLog();
RemoteEditLogManifest manifest = nn.getEditLogManifest(1);
RemoteEditLog log = manifest.getLogs().get(0);
NNStorage dstImage = Mockito.mock(NNStorage.class);
Mockito.doReturn(Lists.newArrayList(new File("/wont-be-written")))
.when(dstImage).getFiles(
Mockito.<NameNodeDirType>anyObject(), Mockito.anyString());
Mockito.doReturn(new StorageInfo(1, 1, "X", 1).toColonSeparatedString())
.when(dstImage).toColonSeparatedString();
try {
TransferFsImage.downloadImageToStorage(fsName, 0, dstImage, false);
fail("Storage info was not verified");
} catch (IOException ioe) {
String msg = StringUtils.stringifyException(ioe);
assertTrue(msg, msg.contains("but the secondary expected"));
}
try {
TransferFsImage.downloadEditsToStorage(fsName, log, dstImage);
fail("Storage info was not verified");
} catch (IOException ioe) {
String msg = StringUtils.stringifyException(ioe);
assertTrue(msg, msg.contains("but the secondary expected"));
}
try {
InetSocketAddress fakeAddr = new InetSocketAddress(1);
TransferFsImage.uploadImageFromStorage(fsName, fakeAddr, dstImage, 0);
fail("Storage info was not verified");
} catch (IOException ioe) {
String msg = StringUtils.stringifyException(ioe);
assertTrue(msg, msg.contains("but the secondary expected"));
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test that, if a storage directory is failed when a checkpoint occurs,
* the non-failed storage directory receives the checkpoint.
*/
@SuppressWarnings("deprecation")
public void testCheckpointWithFailedStorageDir() throws Exception {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
File currentDir = null;
Configuration conf = new HdfsConfiguration();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(true).build();
secondary = startSecondaryNameNode(conf);
// Checkpoint once
secondary.doCheckpoint();
// Now primary NN experiences failure of a volume -- fake by
// setting its current dir to a-x permissions
NamenodeProtocols nn = cluster.getNameNodeRpc();
NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
StorageDirectory sd0 = storage.getStorageDir(0);
StorageDirectory sd1 = storage.getStorageDir(1);
currentDir = sd0.getCurrentDir();
currentDir.setExecutable(false);
// Upload checkpoint when NN has a bad storage dir. This should
// succeed and create the checkpoint in the good dir.
secondary.doCheckpoint();
GenericTestUtils.assertExists(
new File(sd1.getCurrentDir(), NNStorage.getImageFileName(2)));
// Restore the good dir
currentDir.setExecutable(true);
nn.restoreFailedStorage("true");
nn.rollEditLog();
// Checkpoint again -- this should upload to both dirs
secondary.doCheckpoint();
assertNNHasCheckpoints(cluster, ImmutableList.of(8));
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
} finally {
if (currentDir != null) {
currentDir.setExecutable(true);
}
if (secondary != null) {
secondary.shutdown();
}
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test case where the NN is configured with a name-only and an edits-only
* dir, with storage-restore turned on. In this case, if the name-only dir
* disappears and comes back, a new checkpoint after it has been restored
* should function correctly.
* @throws Exception
*/
@SuppressWarnings("deprecation")
public void testCheckpointWithSeparateDirsAfterNameFails() throws Exception {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
File currentDir = null;
Configuration conf = new HdfsConfiguration();
File base_dir = new File(MiniDFSCluster.getBaseDirectory());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, true);
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
MiniDFSCluster.getBaseDirectory() + "/name-only");
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
MiniDFSCluster.getBaseDirectory() + "/edits-only");
conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY,
fileAsURI(new File(base_dir, "namesecondary1")).toString());
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(true)
.manageNameDfsDirs(false)
.build();
secondary = startSecondaryNameNode(conf);
// Checkpoint once
secondary.doCheckpoint();
// Now primary NN experiences failure of its only name dir -- fake by
// setting its current dir to a-x permissions
NamenodeProtocols nn = cluster.getNameNodeRpc();
NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
StorageDirectory sd0 = storage.getStorageDir(0);
assertEquals(NameNodeDirType.IMAGE, sd0.getStorageDirType());
currentDir = sd0.getCurrentDir();
currentDir.setExecutable(false);
// Try to upload checkpoint -- this should fail since there are no
// valid storage dirs
try {
secondary.doCheckpoint();
fail("Did not fail to checkpoint when there are no valid storage dirs");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
"No targets in destination storage", ioe);
}
// Restore the good dir
currentDir.setExecutable(true);
nn.restoreFailedStorage("true");
nn.rollEditLog();
// Checkpoint again -- this should upload to the restored name dir
secondary.doCheckpoint();
assertNNHasCheckpoints(cluster, ImmutableList.of(8));
assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
} finally {
if (currentDir != null) {
currentDir.setExecutable(true);
}
if (secondary != null) {
secondary.shutdown();
}
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test that the 2NN triggers a checkpoint after the configurable interval
*/
public void testCheckpointTriggerOnTxnCount() throws Exception {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1);
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
.format(true).build();
FileSystem fs = cluster.getFileSystem();
secondary = startSecondaryNameNode(conf);
Thread t = new Thread(secondary);
t.start();
final NNStorage storage = secondary.getFSImage().getStorage();
// 2NN should checkpoint at startup
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
LOG.info("Waiting for checkpoint txn id to go to 2");
return storage.getMostRecentCheckpointTxId() == 2;
}
}, 200, 15000);
// If we make 10 transactions, it should checkpoint again
for (int i = 0; i < 10; i++) {
fs.mkdirs(new Path("/test" + i));
}
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
LOG.info("Waiting for checkpoint txn id to go > 2");
return storage.getMostRecentCheckpointTxId() > 2;
}
}, 200, 15000);
} finally {
cleanup(secondary);
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test case where the secondary does a checkpoint, then stops for a while.
* In the meantime, the NN saves its image several times, so that the
* logs that connect the 2NN's old checkpoint to the current txid
* get archived. Then, the 2NN tries to checkpoint again.
*/
public void testSecondaryHasVeryOutOfDateImage() throws IOException {
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
Configuration conf = new HdfsConfiguration();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
.format(true).build();
secondary = startSecondaryNameNode(conf);
// Checkpoint once
secondary.doCheckpoint();
// Now primary NN saves namespace 3 times
NamenodeProtocols nn = cluster.getNameNodeRpc();
nn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
for (int i = 0; i < 3; i++) {
nn.saveNamespace();
}
nn.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
// Now the secondary tries to checkpoint again with its
// old image in memory.
secondary.doCheckpoint();
} finally {
if (secondary != null) {
secondary.shutdown();
}
if (cluster != null) {
cluster.shutdown();
}
}
}
public void testCommandLineParsing() throws ParseException {
SecondaryNameNode.CommandLineOpts opts =
new SecondaryNameNode.CommandLineOpts();
opts.parse();
assertNull(opts.getCommand());
opts.parse("-checkpoint");
assertEquals(SecondaryNameNode.CommandLineOpts.Command.CHECKPOINT,
opts.getCommand());
assertFalse(opts.shouldForceCheckpoint());
opts.parse("-checkpoint", "force");
assertEquals(SecondaryNameNode.CommandLineOpts.Command.CHECKPOINT,
opts.getCommand());
assertTrue(opts.shouldForceCheckpoint());
opts.parse("-geteditsize");
assertEquals(SecondaryNameNode.CommandLineOpts.Command.GETEDITSIZE,
opts.getCommand());
opts.parse("-format");
assertTrue(opts.shouldFormat());
try {
opts.parse("-geteditsize", "-checkpoint");
fail("Should have failed bad parsing for two actions");
} catch (ParseException e) {}
try {
opts.parse("-checkpoint", "xx");
fail("Should have failed for bad checkpoint arg");
} catch (ParseException e) {}
}
private void cleanup(SecondaryNameNode snn) {
if (snn != null) {
try {
snn.shutdown();
} catch (Exception e) {
LOG.warn("Could not shut down secondary namenode", e);
}
}
}
/**
* Assert that if any two files have the same name across the 2NNs
* and NN, they should have the same content too.
*/
private void assertParallelFilesInvariant(MiniDFSCluster cluster,
ImmutableList<SecondaryNameNode> secondaries) throws Exception {
List<File> allCurrentDirs = Lists.newArrayList();
allCurrentDirs.addAll(getNameNodeCurrentDirs(cluster));
for (SecondaryNameNode snn : secondaries) {
allCurrentDirs.addAll(getCheckpointCurrentDirs(snn));
}
FSImageTestUtil.assertParallelFilesAreIdentical(allCurrentDirs,
ImmutableSet.of("VERSION"));
}
private List<File> getCheckpointCurrentDirs(SecondaryNameNode secondary) {
List<File> ret = Lists.newArrayList();
for (URI u : secondary.getCheckpointDirs()) {
File checkpointDir = new File(u.getPath());
ret.add(new File(checkpointDir, "current"));
}
return ret;
}
private CheckpointStorage spyOnSecondaryImage(SecondaryNameNode secondary1) {
CheckpointStorage spy = Mockito.spy((CheckpointStorage)secondary1.getFSImage());;
secondary1.setFSImage(spy);
return spy;
}
/**
* A utility class to perform a checkpoint in a different thread.
*/
private static class DoCheckpointThread extends Thread {
private final SecondaryNameNode snn;
private volatile Throwable thrown = null;
DoCheckpointThread(SecondaryNameNode snn) {
this.snn = snn;
}
@Override
public void run() {
try {
snn.doCheckpoint();
} catch (Throwable t) {
thrown = t;
}
}
void propagateExceptions() {
if (thrown != null) {
throw new RuntimeException(thrown);
}
}
}
}