/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.*;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getInProgressEditsFileName;
import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getFinalizedEditsFileName;
import static org.apache.hadoop.hdfs.server.namenode.NNStorage.getImageFileName;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile;
import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.TransactionalLoadPlan;
import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.LogGroup;
import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.LoadPlan;
import org.junit.Test;
import org.mockito.Mockito;
public class TestFSImageStorageInspector {
private static final Log LOG = LogFactory.getLog(
TestFSImageStorageInspector.class);
/**
* Simple test with image, edits, and inprogress edits
*/
@Test
public void testCurrentStorageInspector() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.IMAGE_AND_EDITS,
false,
"/foo/current/" + getImageFileName(123),
"/foo/current/" + getFinalizedEditsFileName(123, 456),
"/foo/current/" + getImageFileName(456),
"/foo/current/" + getInProgressEditsFileName(457));
inspector.inspectDirectory(mockDir);
mockLogValidation(inspector,
"/foo/current/" + getInProgressEditsFileName(457), 10);
assertEquals(2, inspector.foundEditLogs.size());
assertEquals(2, inspector.foundImages.size());
assertTrue(inspector.foundEditLogs.get(1).isInProgress());
FSImageFile latestImage = inspector.getLatestImage();
assertEquals(456, latestImage.txId);
assertSame(mockDir, latestImage.sd);
assertTrue(inspector.isUpgradeFinalized());
LoadPlan plan = inspector.createLoadPlan();
LOG.info("Plan: " + plan);
assertEquals(new File("/foo/current/"+getImageFileName(456)),
plan.getImageFile());
assertArrayEquals(new File[] {
new File("/foo/current/" + getInProgressEditsFileName(457)) },
plan.getEditsFiles().toArray(new File[0]));
}
/**
* Test that we check for gaps in txids when devising a load plan.
*/
@Test
public void testPlanWithGaps() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.IMAGE_AND_EDITS,
false,
"/foo/current/" + getImageFileName(123),
"/foo/current/" + getImageFileName(456),
"/foo/current/" + getFinalizedEditsFileName(457,900),
"/foo/current/" + getFinalizedEditsFileName(901,950),
"/foo/current/" + getFinalizedEditsFileName(952,1000)); // <-- missing edit 951!
inspector.inspectDirectory(mockDir);
try {
inspector.createLoadPlan();
fail("Didn't throw IOE trying to load with gaps in edits");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains(
"would start at txid 951 but starts at txid 952"));
}
}
/**
* Test the case where an in-progress log comes in the middle of a sequence
* of logs
*/
@Test
public void testPlanWithInProgressInMiddle() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
StorageDirectory mockDir = FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.IMAGE_AND_EDITS,
false,
"/foo/current/" + getImageFileName(123),
"/foo/current/" + getImageFileName(456),
"/foo/current/" + getFinalizedEditsFileName(457,900),
"/foo/current/" + getInProgressEditsFileName(901), // <-- inprogress in middle
"/foo/current/" + getFinalizedEditsFileName(952,1000));
inspector.inspectDirectory(mockDir);
mockLogValidation(inspector,
"/foo/current/" + getInProgressEditsFileName(901), 51);
LoadPlan plan = inspector.createLoadPlan();
LOG.info("Plan: " + plan);
assertEquals(new File("/foo/current/" + getImageFileName(456)),
plan.getImageFile());
assertArrayEquals(new File[] {
new File("/foo/current/" + getFinalizedEditsFileName(457,900)),
new File("/foo/current/" + getInProgressEditsFileName(901)),
new File("/foo/current/" + getFinalizedEditsFileName(952,1000)) },
plan.getEditsFiles().toArray(new File[0]));
}
/**
* Test case for the usual case where no recovery of a log group is necessary
* (i.e all logs have the same start and end txids and finalized)
*/
@Test
public void testLogGroupRecoveryNoop() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo1/current/"
+ getFinalizedEditsFileName(123,456)));
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo2/current/"
+ getFinalizedEditsFileName(123,456)));
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo3/current/"
+ getFinalizedEditsFileName(123,456)));
LogGroup lg = inspector.logGroups.get(123L);
assertEquals(3, lg.logs.size());
lg.planRecovery();
assertFalse(lg.logs.get(0).isCorrupt());
assertFalse(lg.logs.get(1).isCorrupt());
assertFalse(lg.logs.get(2).isCorrupt());
}
/**
* Test case where we have some in-progress and some finalized logs
* for a given txid.
*/
@Test
public void testLogGroupRecoveryMixed() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo1/current/"
+ getFinalizedEditsFileName(123,456)));
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo2/current/"
+ getFinalizedEditsFileName(123,456)));
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo3/current/"
+ getInProgressEditsFileName(123)));
inspector.inspectDirectory(FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.IMAGE,
false,
"/foo4/current/" + getImageFileName(122)));
LogGroup lg = inspector.logGroups.get(123L);
assertEquals(3, lg.logs.size());
EditLogFile inProgressLog = lg.logs.get(2);
assertTrue(inProgressLog.isInProgress());
LoadPlan plan = inspector.createLoadPlan();
// Check that it was marked corrupt.
assertFalse(lg.logs.get(0).isCorrupt());
assertFalse(lg.logs.get(1).isCorrupt());
assertTrue(lg.logs.get(2).isCorrupt());
// Calling recover should move it aside
inProgressLog = spy(inProgressLog);
Mockito.doNothing().when(inProgressLog).moveAsideCorruptFile();
lg.logs.set(2, inProgressLog);
plan.doRecovery();
Mockito.verify(inProgressLog).moveAsideCorruptFile();
}
/**
* Test case where we have finalized logs with different end txids
*/
@Test
public void testLogGroupRecoveryInconsistentEndTxIds() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo1/current/"
+ getFinalizedEditsFileName(123,456)));
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo2/current/"
+ getFinalizedEditsFileName(123,678)));
LogGroup lg = inspector.logGroups.get(123L);
assertEquals(2, lg.logs.size());
try {
lg.planRecovery();
fail("Didn't throw IOE on inconsistent end txids");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains("More than one ending txid"));
}
}
/**
* Test case where we have only in-progress logs and need to synchronize
* based on valid length.
*/
@Test
public void testLogGroupRecoveryInProgress() throws IOException {
String paths[] = new String[] {
"/foo1/current/" + getInProgressEditsFileName(123),
"/foo2/current/" + getInProgressEditsFileName(123),
"/foo3/current/" + getInProgressEditsFileName(123)
};
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[0]));
inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[1]));
inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[2]));
// Inject spies to return the valid counts we would like to see
mockLogValidation(inspector, paths[0], 2000);
mockLogValidation(inspector, paths[1], 2000);
mockLogValidation(inspector, paths[2], 1000);
LogGroup lg = inspector.logGroups.get(123L);
assertEquals(3, lg.logs.size());
lg.planRecovery();
// Check that the short one was marked corrupt
assertFalse(lg.logs.get(0).isCorrupt());
assertFalse(lg.logs.get(1).isCorrupt());
assertTrue(lg.logs.get(2).isCorrupt());
// Calling recover should move it aside
EditLogFile badLog = lg.logs.get(2);
Mockito.doNothing().when(badLog).moveAsideCorruptFile();
Mockito.doNothing().when(lg.logs.get(0)).finalizeLog();
Mockito.doNothing().when(lg.logs.get(1)).finalizeLog();
lg.recover();
Mockito.verify(badLog).moveAsideCorruptFile();
Mockito.verify(lg.logs.get(0)).finalizeLog();
Mockito.verify(lg.logs.get(1)).finalizeLog();
}
/**
* Mock out the log at the given path to return a specified number
* of transactions upon validation.
*/
private void mockLogValidation(
FSImageTransactionalStorageInspector inspector,
String path, int numValidTransactions) throws IOException {
for (LogGroup lg : inspector.logGroups.values()) {
List<EditLogFile> logs = lg.logs;
for (int i = 0; i < logs.size(); i++) {
EditLogFile log = logs.get(i);
if (log.getFile().getPath().equals(path)) {
// mock out its validation
EditLogFile spyLog = spy(log);
doReturn(new FSEditLogLoader.EditLogValidation(-1, numValidTransactions))
.when(spyLog).validateLog();
logs.set(i, spyLog);
return;
}
}
}
fail("No log found to mock out at " + path);
}
/**
* Test when edits and image are in separate directories.
*/
@Test
public void testCurrentSplitEditsAndImage() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
StorageDirectory mockImageDir = FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.IMAGE,
false,
"/foo/current/" + getImageFileName(123));
StorageDirectory mockImageDir2 = FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.IMAGE,
false,
"/foo2/current/" + getImageFileName(456));
StorageDirectory mockEditsDir = FSImageTestUtil.mockStorageDirectory(
NameNodeDirType.EDITS,
false,
"/foo3/current/" + getFinalizedEditsFileName(123, 456),
"/foo3/current/" + getInProgressEditsFileName(457));
inspector.inspectDirectory(mockImageDir);
inspector.inspectDirectory(mockEditsDir);
inspector.inspectDirectory(mockImageDir2);
mockLogValidation(inspector,
"/foo3/current/" + getInProgressEditsFileName(457), 2);
assertEquals(2, inspector.foundEditLogs.size());
assertEquals(2, inspector.foundImages.size());
assertTrue(inspector.foundEditLogs.get(1).isInProgress());
assertTrue(inspector.isUpgradeFinalized());
// Check plan
TransactionalLoadPlan plan =
(TransactionalLoadPlan)inspector.createLoadPlan();
FSImageFile pickedImage = plan.image;
assertEquals(456, pickedImage.txId);
assertSame(mockImageDir2, pickedImage.sd);
assertEquals(new File("/foo2/current/" + getImageFileName(456)),
plan.getImageFile());
assertArrayEquals(new File[] {
new File("/foo3/current/" + getInProgressEditsFileName(457))
}, plan.getEditsFiles().toArray(new File[0]));
}
/**
* Test case where an in-progress log is in an earlier name directory
* than a finalized log. Previously, getEditLogManifest wouldn't
* see this log.
*/
@Test
public void testLogManifestInProgressComesFirst() throws IOException {
FSImageTransactionalStorageInspector inspector =
new FSImageTransactionalStorageInspector();
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo1/current/"
+ getFinalizedEditsFileName(2622,2623),
"/foo1/current/"
+ getFinalizedEditsFileName(2624,2625),
"/foo1/current/"
+ getInProgressEditsFileName(2626)));
inspector.inspectDirectory(
mockDirectoryWithEditLogs("/foo2/current/"
+ getFinalizedEditsFileName(2622,2623),
"/foo2/current/"
+ getFinalizedEditsFileName(2624,2625),
"/foo2/current/"
+ getFinalizedEditsFileName(2626,2627),
"/foo2/current/"
+ getFinalizedEditsFileName(2628,2629)));
}
static StorageDirectory mockDirectoryWithEditLogs(String... fileNames) {
return FSImageTestUtil.mockStorageDirectory(NameNodeDirType.EDITS, false, fileNames);
}
}