/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.bookkeeper.bookie;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
import org.apache.bookkeeper.util.IOUtils;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assert.*;
public class EntryLogTest {
private final static Logger LOG = LoggerFactory.getLogger(EntryLogTest.class);
final List<File> tempDirs = new ArrayList<File>();
File createTempDir(String prefix, String suffix) throws IOException {
File dir = IOUtils.createTempDir(prefix, suffix);
tempDirs.add(dir);
return dir;
}
@After
public void tearDown() throws Exception {
for (File dir : tempDirs) {
FileUtils.deleteDirectory(dir);
}
tempDirs.clear();
}
@Test(timeout=60000)
public void testCorruptEntryLog() throws Exception {
File tmpDir = createTempDir("bkTest", ".dir");
File curDir = Bookie.getCurrentDirectory(tmpDir);
Bookie.checkDirectoryStructure(curDir);
int gcWaitTime = 1000;
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setGcWaitTime(gcWaitTime);
conf.setLedgerDirNames(new String[] {tmpDir.toString()});
Bookie bookie = new Bookie(conf);
// create some entries
EntryLogger logger = ((InterleavedLedgerStorage)bookie.ledgerStorage).entryLogger;
logger.addEntry(1, generateEntry(1, 1));
logger.addEntry(3, generateEntry(3, 1));
logger.addEntry(2, generateEntry(2, 1));
logger.flush();
// now lets truncate the file to corrupt the last entry, which simulates a partial write
File f = new File(curDir, "0.log");
RandomAccessFile raf = new RandomAccessFile(f, "rw");
raf.setLength(raf.length()-10);
raf.close();
// now see which ledgers are in the log
logger = new EntryLogger(conf, bookie.getLedgerDirsManager());
EntryLogMetadata meta = logger.getEntryLogMetadata(0L);
LOG.info("Extracted Meta From Entry Log {}", meta);
assertNotNull(meta.getLedgersMap().get(1L));
assertNull(meta.getLedgersMap().get(2L));
assertNotNull(meta.getLedgersMap().get(3L));
}
private ByteBuffer generateEntry(long ledger, long entry) {
byte[] data = ("ledger-" + ledger + "-" + entry).getBytes();
ByteBuffer bb = ByteBuffer.wrap(new byte[8 + 8 + data.length]);
bb.putLong(ledger);
bb.putLong(entry);
bb.put(data);
bb.flip();
return bb;
}
@Test(timeout=60000)
public void testMissingLogId() throws Exception {
File tmpDir = createTempDir("entryLogTest", ".dir");
File curDir = Bookie.getCurrentDirectory(tmpDir);
Bookie.checkDirectoryStructure(curDir);
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setLedgerDirNames(new String[] {tmpDir.toString()});
Bookie bookie = new Bookie(conf);
// create some entries
int numLogs = 3;
int numEntries = 10;
long[][] positions = new long[2*numLogs][];
for (int i=0; i<numLogs; i++) {
positions[i] = new long[numEntries];
EntryLogger logger = new EntryLogger(conf,
bookie.getLedgerDirsManager());
for (int j=0; j<numEntries; j++) {
positions[i][j] = logger.addEntry(i, generateEntry(i, j));
}
logger.flush();
}
// delete last log id
File lastLogId = new File(curDir, "lastId");
lastLogId.delete();
// write another entries
for (int i=numLogs; i<2*numLogs; i++) {
positions[i] = new long[numEntries];
EntryLogger logger = new EntryLogger(conf,
bookie.getLedgerDirsManager());
for (int j=0; j<numEntries; j++) {
positions[i][j] = logger.addEntry(i, generateEntry(i, j));
}
logger.flush();
}
EntryLogger newLogger = new EntryLogger(conf,
bookie.getLedgerDirsManager());
for (int i=0; i<(2*numLogs+1); i++) {
File logFile = new File(curDir, Long.toHexString(i) + ".log");
assertTrue(logFile.exists());
}
for (int i=0; i<2*numLogs; i++) {
for (int j=0; j<numEntries; j++) {
String expectedValue = "ledger-" + i + "-" + j;
byte[] value = newLogger.readEntry(i, j, positions[i][j]);
ByteBuffer buf = ByteBuffer.wrap(value);
long ledgerId = buf.getLong();
long entryId = buf.getLong();
byte[] data = new byte[buf.remaining()];
buf.get(data);
assertEquals(i, ledgerId);
assertEquals(j, entryId);
assertEquals(expectedValue, new String(data));
}
}
}
@Test(timeout=60000)
/** Test that EntryLogger Should fail with FNFE, if entry logger directories does not exist*/
public void testEntryLoggerShouldThrowFNFEIfDirectoriesDoesNotExist()
throws Exception {
File tmpDir = createTempDir("bkTest", ".dir");
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setLedgerDirNames(new String[] { tmpDir.toString() });
EntryLogger entryLogger = null;
try {
entryLogger = new EntryLogger(conf, new LedgerDirsManager(conf, conf.getLedgerDirs()));
fail("Expecting FileNotFoundException");
} catch (FileNotFoundException e) {
assertEquals("Entry log directory does not exist", e
.getLocalizedMessage());
} finally {
if (entryLogger != null) {
entryLogger.shutdown();
}
}
}
/**
* Test to verify the DiskFull during addEntry
*/
@Test(timeout=60000)
public void testAddEntryFailureOnDiskFull() throws Exception {
File ledgerDir1 = createTempDir("bkTest", ".dir");
File ledgerDir2 = createTempDir("bkTest", ".dir");
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setLedgerDirNames(new String[] { ledgerDir1.getAbsolutePath(),
ledgerDir2.getAbsolutePath() });
Bookie bookie = new Bookie(conf);
EntryLogger entryLogger = new EntryLogger(conf,
bookie.getLedgerDirsManager());
InterleavedLedgerStorage ledgerStorage = ((InterleavedLedgerStorage) bookie.ledgerStorage);
ledgerStorage.entryLogger = entryLogger;
// Create ledgers
ledgerStorage.setMasterKey(1, "key".getBytes());
ledgerStorage.setMasterKey(2, "key".getBytes());
ledgerStorage.setMasterKey(3, "key".getBytes());
// Add entries
ledgerStorage.addEntry(generateEntry(1, 1));
ledgerStorage.addEntry(generateEntry(2, 1));
// Add entry with disk full failure simulation
bookie.getLedgerDirsManager().addToFilledDirs(entryLogger.currentDir);
ledgerStorage.addEntry(generateEntry(3, 1));
// Verify written entries
Assert.assertTrue(0 == generateEntry(1, 1).compareTo(ledgerStorage.getEntry(1, 1)));
Assert.assertTrue(0 == generateEntry(2, 1).compareTo(ledgerStorage.getEntry(2, 1)));
Assert.assertTrue(0 == generateEntry(3, 1).compareTo(ledgerStorage.getEntry(3, 1)));
}
/**
* Explicitely try to recover using the ledgers map index at the end of the entry log
*/
@Test(timeout=60000)
public void testRecoverFromLedgersMap() throws Exception {
File tmpDir = createTempDir("bkTest", ".dir");
File curDir = Bookie.getCurrentDirectory(tmpDir);
Bookie.checkDirectoryStructure(curDir);
int gcWaitTime = 1000;
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setGcWaitTime(gcWaitTime);
conf.setLedgerDirNames(new String[] {tmpDir.toString()});
Bookie bookie = new Bookie(conf);
// create some entries
EntryLogger logger = ((InterleavedLedgerStorage)bookie.ledgerStorage).entryLogger;
logger.addEntry(1, generateEntry(1, 1));
logger.addEntry(3, generateEntry(3, 1));
logger.addEntry(2, generateEntry(2, 1));
logger.addEntry(1, generateEntry(1, 2));
logger.rollLog();
logger.flushRotatedLogs();
EntryLogMetadata meta = logger.extractEntryLogMetadataFromIndex(0L);
LOG.info("Extracted Meta From Entry Log {}", meta);
assertEquals(60, meta.getLedgersMap().get(1L).longValue());
assertEquals(30, meta.getLedgersMap().get(2L).longValue());
assertEquals(30, meta.getLedgersMap().get(3L).longValue());
assertNull(meta.getLedgersMap().get(4L));
assertEquals(120, meta.getTotalSize());
assertEquals(120, meta.getRemainingSize());
}
/**
* Explicitely try to recover using the ledgers map index at the end of the entry log
*/
@Test(timeout = 60000)
public void testRecoverFromLedgersMapOnV0EntryLog() throws Exception {
File tmpDir = createTempDir("bkTest", ".dir");
File curDir = Bookie.getCurrentDirectory(tmpDir);
Bookie.checkDirectoryStructure(curDir);
int gcWaitTime = 1000;
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setGcWaitTime(gcWaitTime);
conf.setLedgerDirNames(new String[] { tmpDir.toString() });
Bookie bookie = new Bookie(conf);
// create some entries
EntryLogger logger = ((InterleavedLedgerStorage) bookie.ledgerStorage).entryLogger;
logger.addEntry(1, generateEntry(1, 1));
logger.addEntry(3, generateEntry(3, 1));
logger.addEntry(2, generateEntry(2, 1));
logger.addEntry(1, generateEntry(1, 2));
logger.rollLog();
// Rewrite the entry log header to be on V0 format
File f = new File(curDir, "0.log");
RandomAccessFile raf = new RandomAccessFile(f, "rw");
raf.seek(EntryLogger.HEADER_VERSION_POSITION);
// Write zeros to indicate V0 + no ledgers map info
raf.write(new byte[4 + 8]);
raf.close();
// now see which ledgers are in the log
logger = new EntryLogger(conf, bookie.getLedgerDirsManager());
try {
logger.extractEntryLogMetadataFromIndex(0L);
fail("Should not be possible to recover from ledgers map index");
} catch (IOException e) {
// Ok
}
// Public method should succeed by falling back to scanning the file
EntryLogMetadata meta = logger.getEntryLogMetadata(0L);
LOG.info("Extracted Meta From Entry Log {}", meta);
assertEquals(60, meta.getLedgersMap().get(1L).longValue());
assertEquals(30, meta.getLedgersMap().get(2L).longValue());
assertEquals(30, meta.getLedgersMap().get(3L).longValue());
assertNull(meta.getLedgersMap().get(4L));
assertEquals(120, meta.getTotalSize());
assertEquals(120, meta.getRemainingSize());
}
}