/*
* Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
* and the EPL 1.0 (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.lang.Thread.UncaughtExceptionHandler;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.h2.mvstore.Chunk;
import org.h2.mvstore.Cursor;
import org.h2.mvstore.DataUtils;
import org.h2.mvstore.FileStore;
import org.h2.mvstore.MVMap;
import org.h2.mvstore.MVStore;
import org.h2.mvstore.OffHeapStore;
import org.h2.mvstore.type.DataType;
import org.h2.mvstore.type.ObjectDataType;
import org.h2.mvstore.type.StringDataType;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.test.utils.AssertThrows;
/**
* Tests the MVStore.
*/
public class TestMVStore extends TestBase {
/**
* Run just this test.
*
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase test = TestBase.createCaller().init();
test.config.traceTest = true;
test.config.big = true;
test.test();
}
@Override
public void test() throws Exception {
testRemoveMapRollback();
testProvidedFileStoreNotOpenedAndClosed();
testVolatileMap();
testEntrySet();
testCompressEmptyPage();
testCompressed();
testFileFormatExample();
testMaxChunkLength();
testCacheInfo();
testRollback();
testVersionsToKeep();
testVersionsToKeep2();
testRemoveMap();
testIsEmpty();
testOffHeapStorage();
testNewerWriteVersion();
testCompactFully();
testBackgroundExceptionListener();
testOldVersion();
testAtomicOperations();
testWriteBuffer();
testWriteDelay();
testEncryptedFile();
testFileFormatChange();
testRecreateMap();
testRenameMapRollback();
testCustomMapType();
testCacheSize();
testConcurrentOpen();
testFileHeader();
testFileHeaderCorruption();
testIndexSkip();
testMinMaxNextKey();
testStoreVersion();
testIterateOldVersion();
testObjects();
testExample();
testExampleMvcc();
testOpenStoreCloseLoop();
testVersion();
testTruncateFile();
testFastDelete();
testRollbackInMemory();
testRollbackStored();
testMeta();
testInMemory();
testLargeImport();
testBtreeStore();
testCompact();
testCompactMapNotOpen();
testReuseSpace();
testRandom();
testKeyValueClasses();
testIterate();
testCloseTwice();
testSimple();
// longer running tests
testLargerThan2G();
}
private void testRemoveMapRollback() {
MVStore store = new MVStore.Builder().
open();
MVMap<String, String> map = store.openMap("test");
map.put("1", "Hello");
store.commit();
store.removeMap(map);
store.rollback();
assertTrue(store.hasMap("test"));
map = store.openMap("test");
// TODO the data should get back alive
assertNull(map.get("1"));
store.close();
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
store = new MVStore.Builder().
autoCommitDisabled().
fileName(fileName).
open();
map = store.openMap("test");
map.put("1", "Hello");
store.commit();
store.removeMap(map);
store.rollback();
assertTrue(store.hasMap("test"));
map = store.openMap("test");
// TODO the data should get back alive
assertNull(map.get("1"));
store.close();
}
private void testProvidedFileStoreNotOpenedAndClosed() {
final AtomicInteger openClose = new AtomicInteger();
FileStore fileStore = new OffHeapStore() {
@Override
public void open(String fileName, boolean readOnly, char[] encryptionKey) {
openClose.incrementAndGet();
super.open(fileName, readOnly, encryptionKey);
}
@Override
public void close() {
openClose.incrementAndGet();
super.close();
}
};
MVStore store = new MVStore.Builder().
fileStore(fileStore).
open();
store.close();
assertEquals(0, openClose.get());
}
private void testVolatileMap() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore store = new MVStore.Builder().
fileName(fileName).
open();
MVMap<String, String> map = store.openMap("test");
assertFalse(map.isVolatile());
map.setVolatile(true);
assertTrue(map.isVolatile());
map.put("1", "Hello");
assertEquals("Hello", map.get("1"));
assertEquals(1, map.size());
store.close();
store = new MVStore.Builder().
fileName(fileName).
open();
assertTrue(store.hasMap("test"));
map = store.openMap("test");
assertEquals(0, map.size());
store.close();
}
private void testEntrySet() {
MVStore s = new MVStore.Builder().open();
MVMap<Integer, Integer> map;
map = s.openMap("data");
for (int i = 0; i < 20; i++) {
map.put(i, i * 10);
}
int next = 0;
for (Entry<Integer, Integer> e : map.entrySet()) {
assertEquals(next, e.getKey().intValue());
assertEquals(next * 10, e.getValue().intValue());
next++;
}
}
private void testCompressEmptyPage() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore store = new MVStore.Builder().
cacheSize(100).fileName(fileName).
compress().
autoCommitBufferSize(10 * 1024).
open();
MVMap<String, String> map = store.openMap("test");
store.removeMap(map);
store.commit();
store.close();
store = new MVStore.Builder().
compress().
open();
store.close();
}
private void testCompressed() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
long lastSize = 0;
for (int level = 0; level <= 2; level++) {
FileUtils.delete(fileName);
MVStore.Builder builder = new MVStore.Builder().fileName(fileName);
if (level == 1) {
builder.compress();
} else if (level == 2) {
builder.compressHigh();
}
MVStore s = builder.open();
MVMap<String, String> map = s.openMap("data");
String data = new String(new char[1000]).replace((char) 0, 'x');
for (int i = 0; i < 400; i++) {
map.put(data + i, data);
}
s.close();
long size = FileUtils.size(fileName);
if (level > 0) {
assertTrue(size < lastSize);
}
lastSize = size;
s = new MVStore.Builder().fileName(fileName).open();
map = s.openMap("data");
for (int i = 0; i < 400; i++) {
assertEquals(data, map.get(data + i));
}
s.close();
}
}
private void testFileFormatExample() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = MVStore.open(fileName);
MVMap<Integer, String> map = s.openMap("data");
for (int i = 0; i < 400; i++) {
map.put(i, "Hello");
}
s.commit();
for (int i = 0; i < 100; i++) {
map.put(0, "Hi");
}
s.commit();
s.close();
// ;MVStoreTool.dump(fileName);
}
private void testMaxChunkLength() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().fileName(fileName).open();
MVMap<Integer, byte[]> map = s.openMap("data");
map.put(0, new byte[2 * 1024 * 1024]);
s.commit();
map.put(1, new byte[10 * 1024]);
s.commit();
MVMap<String, String> meta = s.getMetaMap();
Chunk c = Chunk.fromString(meta.get("chunk.1"));
assertTrue(c.maxLen < Integer.MAX_VALUE);
assertTrue(c.maxLenLive < Integer.MAX_VALUE);
s.close();
}
private void testCacheInfo() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().fileName(fileName).cacheSize(2).open();
assertEquals(2, s.getCacheSize());
MVMap<Integer, byte[]> map;
map = s.openMap("data");
byte[] data = new byte[1024];
for (int i = 0; i < 1000; i++) {
map.put(i, data);
s.commit();
if (i < 50) {
assertEquals(0, s.getCacheSizeUsed());
} else if (i > 300) {
assertTrue(s.getCacheSizeUsed() >= 1);
}
}
s.close();
s = new MVStore.Builder().open();
assertEquals(0, s.getCacheSize());
assertEquals(0, s.getCacheSizeUsed());
s.close();
}
private void testVersionsToKeep() throws Exception {
MVStore s = new MVStore.Builder().open();
MVMap<Integer, Integer> map;
map = s.openMap("data");
for (int i = 0; i < 20; i++) {
long version = s.getCurrentVersion();
map.put(i, i);
s.commit();
if (version >= 6) {
map.openVersion(version - 5);
try {
map.openVersion(version - 6);
fail();
} catch (IllegalArgumentException e) {
// expected
}
}
}
}
private void testVersionsToKeep2() {
MVStore s = new MVStore.Builder().autoCommitDisabled().open();
s.setVersionsToKeep(2);
final MVMap<Integer, String> m = s.openMap("data");
s.commit();
assertEquals(1, s.getCurrentVersion());
m.put(1, "version 1");
s.commit();
assertEquals(2, s.getCurrentVersion());
m.put(1, "version 2");
s.commit();
assertEquals(3, s.getCurrentVersion());
m.put(1, "version 3");
s.commit();
m.put(1, "version 4");
assertEquals("version 4", m.openVersion(4).get(1));
assertEquals("version 3", m.openVersion(3).get(1));
assertEquals("version 2", m.openVersion(2).get(1));
new AssertThrows(IllegalArgumentException.class) {
@Override
public void test() throws Exception {
m.openVersion(1);
}
};
s.close();
}
private void testRemoveMap() throws Exception {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
fileName(fileName).
open();
MVMap<Integer, Integer> map;
map = s.openMap("data");
map.put(1, 1);
assertEquals(1, map.get(1).intValue());
s.commit();
s.removeMap(map);
s.commit();
map = s.openMap("data");
assertTrue(map.isEmpty());
map.put(2, 2);
s.close();
}
private void testIsEmpty() throws Exception {
MVStore s = new MVStore.Builder().
pageSplitSize(50).
open();
Map<Integer, byte[]> m = s.openMap("data");
m.put(1, new byte[50]);
m.put(2, new byte[50]);
m.put(3, new byte[50]);
m.remove(1);
m.remove(2);
m.remove(3);
assertEquals(0, m.size());
assertTrue(m.isEmpty());
s.close();
}
private void testOffHeapStorage() throws Exception {
OffHeapStore offHeap = new OffHeapStore();
MVStore s = new MVStore.Builder().
fileStore(offHeap).
open();
int count = 1000;
Map<Integer, String> map = s.openMap("data");
for (int i = 0; i < count; i++) {
map.put(i, "Hello " + i);
s.commit();
}
assertTrue(offHeap.getWriteCount() > count);
s.close();
s = new MVStore.Builder().
fileStore(offHeap).
open();
map = s.openMap("data");
for (int i = 0; i < count; i++) {
assertEquals("Hello " + i, map.get(i));
}
s.close();
}
private void testNewerWriteVersion() throws Exception {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
encryptionKey("007".toCharArray()).
fileName(fileName).
open();
s.setRetentionTime(Integer.MAX_VALUE);
Map<String, Object> header = s.getStoreHeader();
assertEquals("1", header.get("format").toString());
header.put("formatRead", "1");
header.put("format", "2");
forceWriteStoreHeader(s);
MVMap<Integer, String> m = s.openMap("data");
forceWriteStoreHeader(s);
m.put(0, "Hello World");
s.close();
try {
s = new MVStore.Builder().
encryptionKey("007".toCharArray()).
fileName(fileName).
open();
header = s.getStoreHeader();
fail(header.toString());
} catch (IllegalStateException e) {
assertEquals(DataUtils.ERROR_UNSUPPORTED_FORMAT,
DataUtils.getErrorCode(e.getMessage()));
}
s = new MVStore.Builder().
encryptionKey("007".toCharArray()).
readOnly().
fileName(fileName).
open();
assertTrue(s.getFileStore().isReadOnly());
m = s.openMap("data");
assertEquals("Hello World", m.get(0));
s.close();
FileUtils.setReadOnly(fileName);
s = new MVStore.Builder().
encryptionKey("007".toCharArray()).
fileName(fileName).
open();
assertTrue(s.getFileStore().isReadOnly());
m = s.openMap("data");
assertEquals("Hello World", m.get(0));
s.close();
}
private void testCompactFully() throws Exception {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
fileName(fileName).
autoCommitDisabled().
open();
MVMap<Integer, String> m;
for (int i = 0; i < 100; i++) {
m = s.openMap("data" + i);
m.put(0, "Hello World");
s.commit();
}
for (int i = 0; i < 100; i += 2) {
m = s.openMap("data" + i);
s.removeMap(m);
s.commit();
}
long sizeOld = s.getFileStore().size();
s.compactMoveChunks();
long sizeNew = s.getFileStore().size();
assertTrue("old: " + sizeOld + " new: " + sizeNew, sizeNew < sizeOld);
s.close();
}
private void testBackgroundExceptionListener() throws Exception {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
final AtomicReference<Throwable> exRef =
new AtomicReference<Throwable>();
s = new MVStore.Builder().
fileName(fileName).
backgroundExceptionHandler(new UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
exRef.set(e);
}
}).
open();
s.setAutoCommitDelay(10);
MVMap<Integer, String> m;
m = s.openMap("data");
s.getFileStore().getFile().close();
try {
m.put(1, "Hello");
for (int i = 0; i < 200; i++) {
if (exRef.get() != null) {
break;
}
sleep(10);
}
Throwable e = exRef.get();
assertTrue(e != null);
assertEquals(DataUtils.ERROR_WRITING_FAILED,
DataUtils.getErrorCode(e.getMessage()));
} catch (IllegalStateException e) {
// sometimes it is detected right away
assertEquals(DataUtils.ERROR_CLOSED,
DataUtils.getErrorCode(e.getMessage()));
}
s.closeImmediately();
FileUtils.delete(fileName);
}
private void testAtomicOperations() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, byte[]> m;
s = new MVStore.Builder().
fileName(fileName).
open();
m = s.openMap("data");
// putIfAbsent
assertNull(m.putIfAbsent(1, new byte[1]));
assertEquals(1, m.putIfAbsent(1, new byte[2]).length);
assertEquals(1, m.get(1).length);
// replace
assertNull(m.replace(2, new byte[2]));
assertNull(m.get(2));
assertEquals(1, m.replace(1, new byte[2]).length);
assertEquals(2, m.replace(1, new byte[3]).length);
assertEquals(3, m.replace(1, new byte[1]).length);
// replace with oldValue
assertFalse(m.replace(1, new byte[2], new byte[10]));
assertTrue(m.replace(1, new byte[1], new byte[2]));
assertTrue(m.replace(1, new byte[2], new byte[1]));
// remove
assertFalse(m.remove(1, new byte[2]));
assertTrue(m.remove(1, new byte[1]));
s.close();
FileUtils.delete(fileName);
}
private void testWriteBuffer() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, byte[]> m;
byte[] data = new byte[1000];
long lastSize = 0;
int len = 1000;
for (int bs = 0; bs <= 1; bs++) {
s = new MVStore.Builder().
fileName(fileName).
autoCommitBufferSize(bs).
open();
m = s.openMap("data");
for (int i = 0; i < len; i++) {
m.put(i, data);
}
long size = s.getFileStore().size();
assertTrue("last:" + lastSize + " now: " + size, size > lastSize);
lastSize = size;
s.close();
}
s = new MVStore.Builder().
fileName(fileName).
open();
m = s.openMap("data");
assertTrue(m.containsKey(1));
m.put(-1, data);
s.commit();
m.put(-2, data);
s.close();
s = new MVStore.Builder().
fileName(fileName).
open();
m = s.openMap("data");
assertTrue(m.containsKey(-1));
assertTrue(m.containsKey(-2));
s.close();
FileUtils.delete(fileName);
}
private void testWriteDelay() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, String> m;
FileUtils.delete(fileName);
s = new MVStore.Builder().
autoCommitDisabled().
fileName(fileName).open();
m = s.openMap("data");
m.put(1, "1");
s.commit();
s.close();
s = new MVStore.Builder().
autoCommitDisabled().
fileName(fileName).open();
m = s.openMap("data");
assertEquals(1, m.size());
s.close();
FileUtils.delete(fileName);
s = new MVStore.Builder().
fileName(fileName).
open();
m = s.openMap("data");
m.put(1, "Hello");
m.put(2, "World.");
s.commit();
s.close();
s = new MVStore.Builder().
fileName(fileName).
open();
s.setAutoCommitDelay(2);
m = s.openMap("data");
assertEquals("World.", m.get(2));
m.put(2, "World");
s.commit();
long v = s.getCurrentVersion();
long time = System.nanoTime();
m.put(3, "!");
for (int i = 200; i > 0; i--) {
if (s.getCurrentVersion() > v) {
break;
}
long diff = System.nanoTime() - time;
if (diff > TimeUnit.SECONDS.toNanos(1)) {
fail("diff=" + TimeUnit.NANOSECONDS.toMillis(diff));
}
sleep(10);
}
s.closeImmediately();
s = new MVStore.Builder().
fileName(fileName).
open();
m = s.openMap("data");
assertEquals("Hello", m.get(1));
assertEquals("World", m.get(2));
assertEquals("!", m.get(3));
s.close();
FileUtils.delete(fileName);
}
private void testEncryptedFile() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, String> m;
char[] passwordChars = "007".toCharArray();
s = new MVStore.Builder().
fileName(fileName).
encryptionKey(passwordChars).
open();
assertEquals(0, passwordChars[0]);
assertEquals(0, passwordChars[1]);
assertEquals(0, passwordChars[2]);
assertTrue(FileUtils.exists(fileName));
m = s.openMap("test");
m.put(1, "Hello");
assertEquals("Hello", m.get(1));
s.close();
passwordChars = "008".toCharArray();
try {
s = new MVStore.Builder().
fileName(fileName).
encryptionKey(passwordChars).open();
fail();
} catch (IllegalStateException e) {
assertEquals(DataUtils.ERROR_FILE_CORRUPT,
DataUtils.getErrorCode(e.getMessage()));
}
assertEquals(0, passwordChars[0]);
assertEquals(0, passwordChars[1]);
assertEquals(0, passwordChars[2]);
passwordChars = "007".toCharArray();
s = new MVStore.Builder().
fileName(fileName).
encryptionKey(passwordChars).open();
assertEquals(0, passwordChars[0]);
assertEquals(0, passwordChars[1]);
assertEquals(0, passwordChars[2]);
m = s.openMap("test");
assertEquals("Hello", m.get(1));
s.close();
FileUtils.setReadOnly(fileName);
passwordChars = "007".toCharArray();
s = new MVStore.Builder().
fileName(fileName).
encryptionKey(passwordChars).open();
assertTrue(s.getFileStore().isReadOnly());
s.close();
FileUtils.delete(fileName);
assertFalse(FileUtils.exists(fileName));
}
private void testFileFormatChange() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, Integer> m;
s = openStore(fileName);
s.setRetentionTime(Integer.MAX_VALUE);
m = s.openMap("test");
m.put(1, 1);
Map<String, Object> header = s.getStoreHeader();
int format = Integer.parseInt(header.get("format").toString());
assertEquals(1, format);
header.put("format", Integer.toString(format + 1));
forceWriteStoreHeader(s);
s.close();
try {
openStore(fileName).close();
fail();
} catch (IllegalStateException e) {
assertEquals(DataUtils.ERROR_UNSUPPORTED_FORMAT,
DataUtils.getErrorCode(e.getMessage()));
}
FileUtils.delete(fileName);
}
private void testRecreateMap() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, Integer> m = s.openMap("test");
m.put(1, 1);
s.commit();
s.removeMap(m);
s.close();
s = openStore(fileName);
m = s.openMap("test");
assertNull(m.get(1));
s.close();
}
private void testRenameMapRollback() {
MVStore s = openStore(null);
MVMap<Integer, Integer> map;
map = s.openMap("hello");
map.put(1, 10);
long old = s.commit();
s.renameMap(map, "world");
map.put(2, 20);
assertEquals("world", map.getName());
s.rollbackTo(old);
assertEquals("hello", map.getName());
s.rollbackTo(0);
assertTrue(map.isClosed());
s.close();
}
private void testCustomMapType() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
SequenceMap seq = s.openMap("data", new SequenceMap.Builder());
StringBuilder buff = new StringBuilder();
for (long x : seq.keySet()) {
buff.append(x).append(';');
}
assertEquals("1;2;3;4;5;6;7;8;9;10;", buff.toString());
s.close();
}
private void testCacheSize() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, String> map;
s = new MVStore.Builder().
fileName(fileName).
autoCommitDisabled().
compress().open();
map = s.openMap("test");
// add 10 MB of data
for (int i = 0; i < 1024; i++) {
map.put(i, new String(new char[10240]));
}
s.close();
int[] expectedReadsForCacheSize = {
3407, 2590, 1924, 1440, 1330, 956, 918
};
for (int cacheSize = 0; cacheSize <= 6; cacheSize += 4) {
int cacheMB = 1 + 3 * cacheSize;
s = new MVStore.Builder().
fileName(fileName).
cacheSize(cacheMB).open();
assertEquals(cacheMB, s.getCacheSize());
map = s.openMap("test");
for (int i = 0; i < 1024; i += 128) {
for (int j = 0; j < i; j++) {
String x = map.get(j);
assertEquals(10240, x.length());
}
}
long readCount = s.getFileStore().getReadCount();
int expected = expectedReadsForCacheSize[cacheSize];
assertTrue("reads: " + readCount + " expected: " + expected,
Math.abs(100 - (100 * expected / readCount)) < 5);
s.close();
}
}
private void testConcurrentOpen() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().fileName(fileName).open();
try {
MVStore s1 = new MVStore.Builder().fileName(fileName).open();
s1.close();
fail();
} catch (IllegalStateException e) {
// expected
}
try {
MVStore s1 = new MVStore.Builder().fileName(fileName).readOnly().open();
s1.close();
fail();
} catch (IllegalStateException e) {
// expected
}
assertFalse(s.getFileStore().isReadOnly());
s.close();
s = new MVStore.Builder().fileName(fileName).readOnly().open();
assertTrue(s.getFileStore().isReadOnly());
s.close();
}
private void testFileHeader() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
s.setRetentionTime(Integer.MAX_VALUE);
long time = System.currentTimeMillis();
Map<String, Object> m = s.getStoreHeader();
assertEquals("1", m.get("format").toString());
long creationTime = (Long) m.get("created");
assertTrue(Math.abs(time - creationTime) < 100);
m.put("test", "123");
forceWriteStoreHeader(s);
s.close();
s = openStore(fileName);
Object test = s.getStoreHeader().get("test");
assertFalse(test == null);
assertEquals("123", test.toString());
s.close();
}
private static void forceWriteStoreHeader(MVStore s) {
MVMap<Integer, Integer> map = s.openMap("dummy");
map.put(10, 100);
// this is to ensure the file header is overwritten
// the header is written at least every 20 commits
for (int i = 0; i < 30; i++) {
if (i > 5) {
s.setRetentionTime(0);
// ensure that the next save time is different,
// so that blocks can be reclaimed
// (on Windows, resolution is 10 ms)
sleep(1);
}
map.put(10, 110);
s.commit();
}
s.removeMap(map);
s.commit();
}
private static void sleep(long ms) {
// on Windows, need to sleep in some cases,
// mainly because the milliseconds resolution of
// System.currentTimeMillis is 10 ms.
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
// ignore
}
}
private void testFileHeaderCorruption() throws Exception {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
fileName(fileName).pageSplitSize(1000).autoCommitDisabled().open();
s.setRetentionTime(0);
MVMap<Integer, byte[]> map;
map = s.openMap("test");
map.put(0, new byte[100]);
for (int i = 0; i < 10; i++) {
map = s.openMap("test" + i);
map.put(0, new byte[1000]);
s.commit();
}
FileStore fs = s.getFileStore();
long size = fs.getFile().size();
for (int i = 0; i < 100; i++) {
map = s.openMap("test" + i);
s.removeMap(map);
s.commit();
s.compact(100, 1);
if (fs.getFile().size() <= size) {
break;
}
}
// the last chunk is at the end
s.setReuseSpace(false);
map = s.openMap("test2");
map.put(1, new byte[1000]);
s.close();
FilePath f = FilePath.get(fileName);
int blockSize = 4 * 1024;
// test corrupt file headers
for (int i = 0; i <= blockSize; i += blockSize) {
FileChannel fc = f.open("rw");
if (i == 0) {
// corrupt the last block (the end header)
fc.write(ByteBuffer.allocate(256), fc.size() - 256);
}
ByteBuffer buff = ByteBuffer.allocate(4 * 1024);
fc.read(buff, i);
String h = new String(buff.array(), "UTF-8").trim();
int idx = h.indexOf("fletcher:");
int old = Character.digit(h.charAt(idx + "fletcher:".length()), 16);
int bad = (old + 1) & 15;
buff.put(idx + "fletcher:".length(),
(byte) Character.forDigit(bad, 16));
buff.rewind();
fc.write(buff, i);
fc.close();
if (i == 0) {
// if the first header is corrupt, the second
// header should be used
s = openStore(fileName);
map = s.openMap("test");
assertEquals(100, map.get(0).length);
map = s.openMap("test2");
assertFalse(map.containsKey(1));
s.close();
} else {
// both headers are corrupt
try {
s = openStore(fileName);
fail();
} catch (Exception e) {
// expected
}
}
}
}
private void testIndexSkip() {
MVStore s = openStore(null, 4);
MVMap<Integer, Integer> map = s.openMap("test");
for (int i = 0; i < 100; i += 2) {
map.put(i, 10 * i);
}
Cursor<Integer, Integer> c = map.cursor(50);
// skip must reset the root of the cursor
c.skip(10);
for (int i = 70; i < 100; i += 2) {
assertTrue(c.hasNext());
assertEquals(i, c.next().intValue());
}
assertFalse(c.hasNext());
for (int i = -1; i < 100; i++) {
long index = map.getKeyIndex(i);
if (i < 0 || (i % 2) != 0) {
assertEquals(i < 0 ? -1 : -(i / 2) - 2, index);
} else {
assertEquals(i / 2, index);
}
}
for (int i = -1; i < 60; i++) {
Integer k = map.getKey(i);
if (i < 0 || i >= 50) {
assertNull(k);
} else {
assertEquals(i * 2, k.intValue());
}
}
// skip
c = map.cursor(0);
assertTrue(c.hasNext());
assertEquals(0, c.next().intValue());
c.skip(0);
assertEquals(2, c.next().intValue());
c.skip(1);
assertEquals(6, c.next().intValue());
c.skip(20);
assertEquals(48, c.next().intValue());
c = map.cursor(0);
c.skip(20);
assertEquals(40, c.next().intValue());
c = map.cursor(0);
assertEquals(0, c.next().intValue());
assertEquals(12, map.keyList().indexOf(24));
assertEquals(24, map.keyList().get(12).intValue());
assertEquals(-14, map.keyList().indexOf(25));
assertEquals(map.size(), map.keyList().size());
}
private void testMinMaxNextKey() {
MVStore s = openStore(null);
MVMap<Integer, Integer> map = s.openMap("test");
map.put(10, 100);
map.put(20, 200);
assertEquals(10, map.firstKey().intValue());
assertEquals(20, map.lastKey().intValue());
assertEquals(20, map.ceilingKey(15).intValue());
assertEquals(20, map.ceilingKey(20).intValue());
assertEquals(10, map.floorKey(15).intValue());
assertEquals(10, map.floorKey(10).intValue());
assertEquals(20, map.higherKey(10).intValue());
assertEquals(10, map.lowerKey(20).intValue());
final MVMap<Integer, Integer> m = map;
assertEquals(10, m.ceilingKey(null).intValue());
assertEquals(10, m.higherKey(null).intValue());
assertNull(m.lowerKey(null));
assertNull(m.floorKey(null));
for (int i = 3; i < 20; i++) {
s = openStore(null, 4);
map = s.openMap("test");
for (int j = 3; j < i; j++) {
map.put(j * 2, j * 20);
}
if (i == 3) {
assertNull(map.firstKey());
assertNull(map.lastKey());
} else {
assertEquals(6, map.firstKey().intValue());
int max = (i - 1) * 2;
assertEquals(max, map.lastKey().intValue());
for (int j = 0; j < i * 2 + 2; j++) {
if (j > max) {
assertNull(map.ceilingKey(j));
} else {
int ceiling = Math.max((j + 1) / 2 * 2, 6);
assertEquals(ceiling, map.ceilingKey(j).intValue());
}
int floor = Math.min(max, Math.max(j / 2 * 2, 4));
if (floor < 6) {
assertNull(map.floorKey(j));
} else {
map.floorKey(j);
}
int lower = Math.min(max, Math.max((j - 1) / 2 * 2, 4));
if (lower < 6) {
assertNull(map.lowerKey(j));
} else {
assertEquals(lower, map.lowerKey(j).intValue());
}
int higher = Math.max((j + 2) / 2 * 2, 6);
if (higher > max) {
assertNull(map.higherKey(j));
} else {
assertEquals(higher, map.higherKey(j).intValue());
}
}
}
}
}
private void testStoreVersion() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = MVStore.open(fileName);
assertEquals(0, s.getCurrentVersion());
assertEquals(0, s.getStoreVersion());
s.setStoreVersion(0);
s.commit();
s.setStoreVersion(1);
s.closeImmediately();
s = MVStore.open(fileName);
assertEquals(1, s.getCurrentVersion());
assertEquals(0, s.getStoreVersion());
s.setStoreVersion(1);
s.close();
s = MVStore.open(fileName);
assertEquals(2, s.getCurrentVersion());
assertEquals(1, s.getStoreVersion());
s.close();
}
private void testIterateOldVersion() {
MVStore s;
Map<Integer, Integer> map;
s = new MVStore.Builder().open();
map = s.openMap("test");
int len = 100;
for (int i = 0; i < len; i++) {
map.put(i, 10 * i);
}
Iterator<Integer> it = map.keySet().iterator();
s.commit();
for (int i = 0; i < len; i += 2) {
map.remove(i);
}
int count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(len, count);
s.close();
}
private void testObjects() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
Map<Object, Object> map;
s = new MVStore.Builder().fileName(fileName).open();
map = s.openMap("test");
map.put(1, "Hello");
map.put("2", 200);
map.put(new Object[1], new Object[]{1, "2"});
s.close();
s = new MVStore.Builder().fileName(fileName).open();
map = s.openMap("test");
assertEquals("Hello", map.get(1).toString());
assertEquals(200, ((Integer) map.get("2")).intValue());
Object[] x = (Object[]) map.get(new Object[1]);
assertEquals(2, x.length);
assertEquals(1, ((Integer) x[0]).intValue());
assertEquals("2", (String) x[1]);
s.close();
}
private void testExample() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
// open the store (in-memory if fileName is null)
MVStore s = MVStore.open(fileName);
// create/get the map named "data"
MVMap<Integer, String> map = s.openMap("data");
// add and read some data
map.put(1, "Hello World");
// System.out.println(map.get(1));
// close the store (this will persist changes)
s.close();
s = MVStore.open(fileName);
map = s.openMap("data");
assertEquals("Hello World", map.get(1));
s.close();
}
private void testExampleMvcc() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
// open the store (in-memory if fileName is null)
MVStore s = MVStore.open(fileName);
// create/get the map named "data"
MVMap<Integer, String> map = s.openMap("data");
// add some data
map.put(1, "Hello");
map.put(2, "World");
// get the current version, for later use
long oldVersion = s.getCurrentVersion();
// from now on, the old version is read-only
s.commit();
// more changes, in the new version
// changes can be rolled back if required
// changes always go into "head" (the newest version)
map.put(1, "Hi");
map.remove(2);
// access the old data (before the commit)
MVMap<Integer, String> oldMap =
map.openVersion(oldVersion);
// print the old version (can be done
// concurrently with further modifications)
// this will print "Hello" and "World":
// System.out.println(oldMap.get(1));
assertEquals("Hello", oldMap.get(1));
// System.out.println(oldMap.get(2));
assertEquals("World", oldMap.get(2));
// print the newest version ("Hi")
// System.out.println(map.get(1));
assertEquals("Hi", map.get(1));
// close the store
s.close();
}
private void testOpenStoreCloseLoop() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
for (int k = 0; k < 1; k++) {
// long t = System.nanoTime();
for (int j = 0; j < 3; j++) {
MVStore s = openStore(fileName);
Map<String, Integer> m = s.openMap("data");
for (int i = 0; i < 3; i++) {
Integer x = m.get("value");
m.put("value", x == null ? 0 : x + 1);
s.commit();
}
s.close();
}
// System.out.println("open/close: " +
// TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t));
// System.out.println("size: " + FileUtils.size(fileName));
}
}
private void testOldVersion() {
MVStore s;
for (int op = 0; op <= 1; op++) {
for (int i = 0; i < 5; i++) {
s = openStore(null);
s.setVersionsToKeep(Integer.MAX_VALUE);
MVMap<String, String> m;
m = s.openMap("data");
for (int j = 0; j < 5; j++) {
if (op == 1) {
m.put("1", "" + s.getCurrentVersion());
}
s.commit();
}
for (int j = 0; j < s.getCurrentVersion(); j++) {
MVMap<String, String> old = m.openVersion(j);
if (op == 1) {
assertEquals("" + j, old.get("1"));
}
}
s.close();
}
}
}
private void testVersion() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
s = openStore(fileName);
s.setVersionsToKeep(100);
s.setAutoCommitDelay(0);
s.setRetentionTime(Integer.MAX_VALUE);
MVMap<String, String> m = s.openMap("data");
s.commit();
long first = s.getCurrentVersion();
m.put("0", "test");
s.commit();
m.put("1", "Hello");
m.put("2", "World");
for (int i = 10; i < 20; i++) {
m.put("" + i, "data");
}
long old = s.getCurrentVersion();
s.commit();
m.put("1", "Hallo");
m.put("2", "Welt");
MVMap<String, String> mFirst;
mFirst = m.openVersion(first);
assertEquals(0, mFirst.size());
MVMap<String, String> mOld;
assertEquals("Hallo", m.get("1"));
assertEquals("Welt", m.get("2"));
mOld = m.openVersion(old);
assertEquals("Hello", mOld.get("1"));
assertEquals("World", mOld.get("2"));
assertTrue(mOld.isReadOnly());
s.getCurrentVersion();
long old3 = s.commit();
// the old version is still available
assertEquals("Hello", mOld.get("1"));
assertEquals("World", mOld.get("2"));
mOld = m.openVersion(old3);
assertEquals("Hallo", mOld.get("1"));
assertEquals("Welt", mOld.get("2"));
m.put("1", "Hi");
assertEquals("Welt", m.remove("2"));
s.close();
s = openStore(fileName);
m = s.openMap("data");
assertEquals("Hi", m.get("1"));
assertEquals(null, m.get("2"));
mOld = m.openVersion(old3);
assertEquals("Hallo", mOld.get("1"));
assertEquals("Welt", mOld.get("2"));
try {
m.openVersion(-3);
fail();
} catch (IllegalArgumentException e) {
// expected
}
s.close();
}
private void testTruncateFile() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, String> m;
s = openStore(fileName);
m = s.openMap("data");
String data = new String(new char[10000]).replace((char) 0, 'x');
for (int i = 1; i < 10; i++) {
m.put(i, data);
s.commit();
}
s.close();
long len = FileUtils.size(fileName);
s = openStore(fileName);
s.setRetentionTime(0);
// remove 75%
m = s.openMap("data");
for (int i = 0; i < 10; i++) {
if (i % 4 != 0) {
sleep(2);
m.remove(i);
s.commit();
}
}
assertTrue(s.compact(100, 50 * 1024));
s.close();
long len2 = FileUtils.size(fileName);
assertTrue("len2: " + len2 + " len: " + len, len2 < len);
}
private void testFastDelete() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, String> m;
s = openStore(fileName, 700);
m = s.openMap("data");
for (int i = 0; i < 1000; i++) {
m.put(i, "Hello World");
assertEquals(i + 1, m.size());
}
assertEquals(1000, m.size());
assertEquals(131896, s.getUnsavedMemory());
s.commit();
assertEquals(2, s.getFileStore().getWriteCount());
s.close();
s = openStore(fileName);
m = s.openMap("data");
m.clear();
assertEquals(0, m.size());
s.commit();
// ensure only nodes are read, but not leaves
assertEquals(45, s.getFileStore().getReadCount());
assertTrue(s.getFileStore().getWriteCount() < 5);
s.close();
}
private void testRollback() {
MVStore s = MVStore.open(null);
MVMap<Integer, Integer> m = s.openMap("m");
m.put(1, -1);
s.commit();
for (int i = 0; i < 10; i++) {
m.put(1, i);
s.rollback();
assertEquals(i - 1, m.get(1).intValue());
m.put(1, i);
s.commit();
}
}
private void testRollbackStored() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVMap<String, String> meta;
MVStore s = openStore(fileName);
assertEquals(45000, s.getRetentionTime());
s.setRetentionTime(0);
assertEquals(0, s.getRetentionTime());
s.setRetentionTime(45000);
assertEquals(45000, s.getRetentionTime());
assertEquals(0, s.getCurrentVersion());
assertFalse(s.hasUnsavedChanges());
MVMap<String, String> m = s.openMap("data");
assertTrue(s.hasUnsavedChanges());
MVMap<String, String> m0 = s.openMap("data0");
m.put("1", "Hello");
assertEquals(1, s.commit());
s.rollbackTo(1);
assertEquals(1, s.getCurrentVersion());
assertEquals("Hello", m.get("1"));
// so a new version is created
m.put("1", "Hello");
long v2 = s.commit();
assertEquals(2, v2);
assertEquals(2, s.getCurrentVersion());
assertFalse(s.hasUnsavedChanges());
assertEquals("Hello", m.get("1"));
s.close();
s = openStore(fileName);
s.setRetentionTime(45000);
assertEquals(2, s.getCurrentVersion());
meta = s.getMetaMap();
m = s.openMap("data");
assertFalse(s.hasUnsavedChanges());
assertEquals("Hello", m.get("1"));
m0 = s.openMap("data0");
MVMap<String, String> m1 = s.openMap("data1");
m.put("1", "Hallo");
m0.put("1", "Hallo");
m1.put("1", "Hallo");
assertEquals("Hallo", m.get("1"));
assertEquals("Hallo", m1.get("1"));
assertTrue(s.hasUnsavedChanges());
s.rollbackTo(v2);
assertFalse(s.hasUnsavedChanges());
assertNull(meta.get("name.data1"));
assertNull(m0.get("1"));
assertEquals("Hello", m.get("1"));
assertEquals(2, s.commit());
s.close();
s = openStore(fileName);
s.setRetentionTime(45000);
assertEquals(2, s.getCurrentVersion());
meta = s.getMetaMap();
assertTrue(meta.get("name.data") != null);
assertTrue(meta.get("name.data0") != null);
assertNull(meta.get("name.data1"));
m = s.openMap("data");
m0 = s.openMap("data0");
assertNull(m0.get("1"));
assertEquals("Hello", m.get("1"));
assertFalse(m0.isReadOnly());
m.put("1", "Hallo");
s.commit();
long v3 = s.getCurrentVersion();
assertEquals(3, v3);
s.close();
s = openStore(fileName);
s.setRetentionTime(45000);
assertEquals(3, s.getCurrentVersion());
m = s.openMap("data");
m.put("1", "Hi");
s.close();
s = openStore(fileName);
s.setRetentionTime(45000);
m = s.openMap("data");
assertEquals("Hi", m.get("1"));
s.rollbackTo(v3);
assertEquals("Hallo", m.get("1"));
s.close();
s = openStore(fileName);
s.setRetentionTime(45000);
m = s.openMap("data");
assertEquals("Hallo", m.get("1"));
s.close();
}
private void testRollbackInMemory() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName, 5);
s.setAutoCommitDelay(0);
assertEquals(0, s.getCurrentVersion());
MVMap<String, String> m = s.openMap("data");
s.rollbackTo(0);
assertTrue(m.isClosed());
assertEquals(0, s.getCurrentVersion());
m = s.openMap("data");
MVMap<String, String> m0 = s.openMap("data0");
MVMap<String, String> m2 = s.openMap("data2");
m.put("1", "Hello");
for (int i = 0; i < 10; i++) {
m2.put("" + i, "Test");
}
long v1 = s.commit();
assertEquals(1, v1);
assertEquals(1, s.getCurrentVersion());
MVMap<String, String> m1 = s.openMap("data1");
assertEquals("Test", m2.get("1"));
m.put("1", "Hallo");
m0.put("1", "Hallo");
m1.put("1", "Hallo");
m2.clear();
assertEquals("Hallo", m.get("1"));
assertEquals("Hallo", m1.get("1"));
s.rollbackTo(v1);
assertEquals(1, s.getCurrentVersion());
for (int i = 0; i < 10; i++) {
assertEquals("Test", m2.get("" + i));
}
assertEquals("Hello", m.get("1"));
assertNull(m0.get("1"));
assertTrue(m1.isClosed());
assertFalse(m0.isReadOnly());
s.close();
}
private void testMeta() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
s.setRetentionTime(Integer.MAX_VALUE);
MVMap<String, String> m = s.getMetaMap();
assertEquals("[]", s.getMapNames().toString());
MVMap<String, String> data = s.openMap("data");
data.put("1", "Hello");
data.put("2", "World");
s.commit();
assertEquals(1, s.getCurrentVersion());
assertEquals("[data]", s.getMapNames().toString());
assertEquals("data", s.getMapName(data.getId()));
assertNull(s.getMapName(s.getMetaMap().getId()));
assertNull(s.getMapName(data.getId() + 1));
String id = s.getMetaMap().get("name.data");
assertEquals("name:data", m.get("map." + id));
assertEquals("Hello", data.put("1", "Hallo"));
s.commit();
assertEquals("name:data", m.get("map." + id));
assertTrue(m.get("root.1").length() > 0);
assertTrue(m.containsKey("chunk.1"));
assertEquals(2, s.getCurrentVersion());
s.rollbackTo(1);
assertEquals("Hello", data.get("1"));
assertEquals("World", data.get("2"));
s.close();
}
private void testInMemory() {
for (int j = 0; j < 1; j++) {
MVStore s = openStore(null);
// s.setMaxPageSize(10);
int len = 100;
// TreeMap<Integer, String> m = new TreeMap<Integer, String>();
// HashMap<Integer, String> m = New.hashMap();
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < len; i++) {
assertNull(m.put(i, "Hello World"));
}
for (int i = 0; i < len; i++) {
assertEquals("Hello World", m.get(i));
}
for (int i = 0; i < len; i++) {
assertEquals("Hello World", m.remove(i));
}
assertEquals(null, m.get(0));
assertEquals(0, m.size());
s.close();
}
}
private void testLargeImport() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
int len = 1000;
for (int j = 0; j < 5; j++) {
FileUtils.delete(fileName);
MVStore s = openStore(fileName, 40);
MVMap<Integer, Object[]> m = s.openMap("data",
new MVMap.Builder<Integer, Object[]>()
.valueType(new RowDataType(new DataType[] {
new ObjectDataType(),
StringDataType.INSTANCE,
StringDataType.INSTANCE })));
// Profiler prof = new Profiler();
// prof.startCollecting();
// long t = System.nanoTime();
for (int i = 0; i < len;) {
Object[] o = new Object[3];
o[0] = i;
o[1] = "Hello World";
o[2] = "World";
m.put(i, o);
i++;
if (i % 10000 == 0) {
s.commit();
}
}
s.close();
// System.out.println(prof.getTop(5));
// System.out.println("store time " +
// TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t));
// System.out.println("store size " +
// FileUtils.size(fileName));
}
}
private void testBtreeStore() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
s.close();
s = openStore(fileName);
MVMap<Integer, String> m = s.openMap("data");
int count = 2000;
for (int i = 0; i < count; i++) {
assertNull(m.put(i, "hello " + i));
assertEquals("hello " + i, m.get(i));
}
s.commit();
assertEquals("hello 0", m.remove(0));
assertNull(m.get(0));
for (int i = 1; i < count; i++) {
assertEquals("hello " + i, m.get(i));
}
s.close();
s = openStore(fileName);
m = s.openMap("data");
assertNull(m.get(0));
for (int i = 1; i < count; i++) {
assertEquals("hello " + i, m.get(i));
}
for (int i = 1; i < count; i++) {
m.remove(i);
}
s.commit();
assertNull(m.get(0));
for (int i = 0; i < count; i++) {
assertNull(m.get(i));
}
s.close();
}
private void testCompactMapNotOpen() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName, 1000);
MVMap<Integer, String> m = s.openMap("data");
int factor = 100;
for (int j = 0; j < 10; j++) {
for (int i = j * factor; i < 10 * factor; i++) {
m.put(i, "Hello" + j);
}
s.commit();
}
s.close();
s = openStore(fileName);
s.setRetentionTime(0);
Map<String, String> meta = s.getMetaMap();
int chunkCount1 = 0;
for (String k : meta.keySet()) {
if (k.startsWith("chunk.")) {
chunkCount1++;
}
}
s.compact(80, 1);
s.compact(80, 1);
int chunkCount2 = 0;
for (String k : meta.keySet()) {
if (k.startsWith("chunk.")) {
chunkCount2++;
}
}
assertTrue(chunkCount2 >= chunkCount1);
m = s.openMap("data");
for (int i = 0; i < 10; i++) {
sleep(1);
boolean result = s.compact(50, 50 * 1024);
if (!result) {
break;
}
}
assertFalse(s.compact(50, 1024));
int chunkCount3 = 0;
for (String k : meta.keySet()) {
if (k.startsWith("chunk.")) {
chunkCount3++;
}
}
assertTrue(chunkCount1 + ">" + chunkCount2 + ">" + chunkCount3,
chunkCount3 < chunkCount1);
for (int i = 0; i < 10 * factor; i++) {
assertEquals("x" + i, "Hello" + (i / factor), m.get(i));
}
s.close();
}
private void testCompact() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
long initialLength = 0;
for (int j = 0; j < 20; j++) {
sleep(2);
MVStore s = openStore(fileName);
s.setRetentionTime(0);
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < 100; i++) {
m.put(j + i, "Hello " + j);
}
s.compact(80, 1024);
s.close();
long len = FileUtils.size(fileName);
// System.out.println(" len:" + len);
if (initialLength == 0) {
initialLength = len;
} else {
assertTrue("initial: " + initialLength + " len: " + len,
len <= initialLength * 3);
}
}
// long len = FileUtils.size(fileName);
// System.out.println("len0: " + len);
MVStore s = openStore(fileName);
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < 100; i++) {
m.remove(i);
}
s.compact(80, 1024);
s.close();
// len = FileUtils.size(fileName);
// System.out.println("len1: " + len);
s = openStore(fileName);
m = s.openMap("data");
s.compact(80, 1024);
s.close();
// len = FileUtils.size(fileName);
// System.out.println("len2: " + len);
}
private void testReuseSpace() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
long initialLength = 0;
for (int j = 0; j < 20; j++) {
sleep(2);
MVStore s = openStore(fileName);
s.setRetentionTime(0);
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < 10; i++) {
m.put(i, "Hello");
}
s.commit();
for (int i = 0; i < 10; i++) {
assertEquals("Hello", m.get(i));
assertEquals("Hello", m.remove(i));
}
s.close();
long len = FileUtils.size(fileName);
if (initialLength == 0) {
initialLength = len;
} else {
assertTrue("len: " + len + " initial: " + initialLength + " j: " + j,
len <= initialLength * 5);
}
}
}
private void testRandom() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, Integer> m = s.openMap("data");
TreeMap<Integer, Integer> map = new TreeMap<Integer, Integer>();
Random r = new Random(1);
int operationCount = 1000;
int maxValue = 30;
Integer expected, got;
for (int i = 0; i < operationCount; i++) {
int k = r.nextInt(maxValue);
int v = r.nextInt();
boolean compareAll;
switch (r.nextInt(3)) {
case 0:
log(i + ": put " + k + " = " + v);
expected = map.put(k, v);
got = m.put(k, v);
if (expected == null) {
assertNull(got);
} else {
assertEquals(expected, got);
}
compareAll = true;
break;
case 1:
log(i + ": remove " + k);
expected = map.remove(k);
got = m.remove(k);
if (expected == null) {
assertNull(got);
} else {
assertEquals(expected, got);
}
compareAll = true;
break;
default:
Integer a = map.get(k);
Integer b = m.get(k);
if (a == null || b == null) {
assertTrue(a == b);
} else {
assertEquals(a.intValue(), b.intValue());
}
compareAll = false;
break;
}
if (compareAll) {
Iterator<Integer> it = m.keyIterator(null);
Iterator<Integer> itExpected = map.keySet().iterator();
while (itExpected.hasNext()) {
assertTrue(it.hasNext());
expected = itExpected.next();
got = it.next();
assertEquals(expected, got);
}
assertFalse(it.hasNext());
}
}
s.close();
}
private void testKeyValueClasses() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, String> is = s.openMap("intString");
is.put(1, "Hello");
MVMap<Integer, Integer> ii = s.openMap("intInt");
ii.put(1, 10);
MVMap<String, Integer> si = s.openMap("stringInt");
si.put("Test", 10);
MVMap<String, String> ss = s.openMap("stringString");
ss.put("Hello", "World");
s.close();
s = openStore(fileName);
is = s.openMap("intString");
assertEquals("Hello", is.get(1));
ii = s.openMap("intInt");
assertEquals(10, ii.get(1).intValue());
si = s.openMap("stringInt");
assertEquals(10, si.get("Test").intValue());
ss = s.openMap("stringString");
assertEquals("World", ss.get("Hello"));
s.close();
}
private void testIterate() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, String> m = s.openMap("data");
Iterator<Integer> it = m.keyIterator(null);
assertFalse(it.hasNext());
for (int i = 0; i < 10; i++) {
m.put(i, "hello " + i);
}
s.commit();
it = m.keyIterator(null);
it.next();
assertThrows(UnsupportedOperationException.class, it).remove();
it = m.keyIterator(null);
for (int i = 0; i < 10; i++) {
assertTrue(it.hasNext());
assertEquals(i, it.next().intValue());
}
assertFalse(it.hasNext());
assertNull(it.next());
for (int j = 0; j < 10; j++) {
it = m.keyIterator(j);
for (int i = j; i < 10; i++) {
assertTrue(it.hasNext());
assertEquals(i, it.next().intValue());
}
assertFalse(it.hasNext());
}
s.close();
}
private void testCloseTwice() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < 3; i++) {
m.put(i, "hello " + i);
}
// closing twice should be fine
s.close();
s.close();
}
private void testSimple() {
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < 3; i++) {
m.put(i, "hello " + i);
}
s.commit();
assertEquals("hello 0", m.remove(0));
assertNull(m.get(0));
for (int i = 1; i < 3; i++) {
assertEquals("hello " + i, m.get(i));
}
s.close();
s = openStore(fileName);
m = s.openMap("data");
assertNull(m.get(0));
for (int i = 1; i < 3; i++) {
assertEquals("hello " + i, m.get(i));
}
s.close();
}
private void testLargerThan2G() {
if (!config.big) {
return;
}
String fileName = getBaseDir() + "/" + getTestName();
FileUtils.delete(fileName);
MVStore store = new MVStore.Builder().cacheSize(16).
fileName(fileName).open();
try {
MVMap<Integer, String> map = store.openMap("test");
long last = System.nanoTime();
String data = new String(new char[2500]).replace((char) 0, 'x');
for (int i = 0;; i++) {
map.put(i, data);
if (i % 10000 == 0) {
store.commit();
long time = System.nanoTime();
if (time - last > TimeUnit.SECONDS.toNanos(2)) {
long mb = store.getFileStore().size() / 1024 / 1024;
trace(mb + "/4500");
if (mb > 4500) {
break;
}
last = time;
}
}
}
store.commit();
store.close();
} finally {
store.closeImmediately();
}
FileUtils.delete(fileName);
}
/**
* Open a store for the given file name, using a small page size.
*
* @param fileName the file name (null for in-memory)
* @return the store
*/
protected static MVStore openStore(String fileName) {
return openStore(fileName, 1000);
}
/**
* Open a store for the given file name, using a small page size.
*
* @param fileName the file name (null for in-memory)
* @param pageSplitSize the page split size
* @return the store
*/
protected static MVStore openStore(String fileName, int pageSplitSize) {
MVStore store = new MVStore.Builder().
fileName(fileName).pageSplitSize(pageSplitSize).open();
return store;
}
/**
* Log the message.
*
* @param msg the message
*/
protected static void log(String msg) {
// System.out.println(msg);
}
}