/*
* Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
* and the EPL 1.0 (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.h2.mvstore.DataUtils;
import org.h2.mvstore.MVMap;
import org.h2.mvstore.MVStore;
import org.h2.mvstore.StreamStore;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.util.IOUtils;
import org.h2.util.New;
import org.h2.util.StringUtils;
/**
* Test the stream store.
*/
public class TestStreamStore extends TestBase {
/**
* Run just this test.
*
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
}
@Override
public void test() throws IOException {
testMaxBlockKey();
testIOException();
testSaveCount();
testExceptionDuringStore();
testReadCount();
testLarge();
testDetectIllegalId();
testTreeStructure();
testFormat();
testWithExistingData();
testWithFullMap();
testLoop();
}
private void testMaxBlockKey() throws IOException {
TreeMap<Long, byte[]> map = new TreeMap<Long, byte[]>();
StreamStore s = new StreamStore(map);
s.setMaxBlockSize(128);
s.setMinBlockSize(64);
map.clear();
for (int len = 1; len < 1024 * 1024; len *= 2) {
byte[] id = s.put(new ByteArrayInputStream(new byte[len]));
long max = s.getMaxBlockKey(id);
if (max == -1) {
assertTrue(map.isEmpty());
} else {
assertEquals(map.lastKey(), (Long) max);
}
}
}
private void testIOException() throws IOException {
HashMap<Long, byte[]> map = New.hashMap();
StreamStore s = new StreamStore(map);
byte[] id = s.put(new ByteArrayInputStream(new byte[1024 * 1024]));
InputStream in = s.get(id);
map.clear();
try {
while (true) {
if (in.read() < 0) {
break;
}
}
fail();
} catch (IOException e) {
assertEquals(DataUtils.ERROR_BLOCK_NOT_FOUND,
DataUtils.getErrorCode(e.getMessage()));
}
}
private void testSaveCount() throws IOException {
String fileName = getBaseDir() + "/testSaveCount.h3";
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
fileName(fileName).
open();
MVMap<Long, byte[]> map = s.openMap("data");
StreamStore streamStore = new StreamStore(map);
int blockSize = 256 * 1024;
assertEquals(blockSize, streamStore.getMaxBlockSize());
for (int i = 0; i < 8 * 16; i++) {
streamStore.put(new RandomStream(blockSize, i));
}
long writeCount = s.getFileStore().getWriteCount();
assertTrue(writeCount > 2);
s.close();
}
private void testExceptionDuringStore() throws IOException {
// test that if there is an IOException while storing
// the data, the entries in the map are "rolled back"
HashMap<Long, byte[]> map = New.hashMap();
StreamStore s = new StreamStore(map);
s.setMaxBlockSize(1024);
assertThrows(IOException.class, s).
put(createFailingStream(new IOException()));
assertEquals(0, map.size());
// the runtime exception is converted to an IOException
assertThrows(IOException.class, s).
put(createFailingStream(new IllegalStateException()));
assertEquals(0, map.size());
}
private void testReadCount() throws IOException {
String fileName = getBaseDir() + "/testReadCount.h3";
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
fileName(fileName).
open();
s.setCacheSize(1);
StreamStore streamStore = getAutoCommitStreamStore(s);
long size = s.getPageSplitSize() * 2;
for (int i = 0; i < 100; i++) {
streamStore.put(new RandomStream(size, i));
}
s.commit();
MVMap<Long, byte[]> map = s.openMap("data");
assertTrue("size: " + map.size(), map.sizeAsLong() >= 100);
s.close();
s = new MVStore.Builder().
fileName(fileName).
open();
streamStore = getAutoCommitStreamStore(s);
for (int i = 0; i < 100; i++) {
streamStore.put(new RandomStream(size, -i));
}
s.commit();
long readCount = s.getFileStore().getReadCount();
// the read count should be low because new blocks
// are appended at the end (not between existing blocks)
assertTrue("rc: " + readCount, readCount < 15);
map = s.openMap("data");
assertTrue("size: " + map.size(), map.sizeAsLong() >= 200);
s.close();
}
private static StreamStore getAutoCommitStreamStore(final MVStore s) {
MVMap<Long, byte[]> map = s.openMap("data");
return new StreamStore(map) {
@Override
protected void onStore(int len) {
if (s.getUnsavedMemory() > s.getAutoCommitMemory() / 2) {
s.commit();
}
}
};
}
private void testLarge() throws IOException {
String fileName = getBaseDir() + "/testVeryLarge.h3";
FileUtils.delete(fileName);
final MVStore s = new MVStore.Builder().
fileName(fileName).
open();
MVMap<Long, byte[]> map = s.openMap("data");
final AtomicInteger count = new AtomicInteger();
StreamStore streamStore = new StreamStore(map) {
@Override
protected void onStore(int len) {
count.incrementAndGet();
s.commit();
}
};
long size = 1 * 1024 * 1024;
streamStore.put(new RandomStream(size, 0));
s.close();
assertEquals(4, count.get());
}
/**
* A stream of incompressible data.
*/
static class RandomStream extends InputStream {
private long pos, size;
private int seed;
RandomStream(long size, int seed) {
this.size = size;
this.seed = seed;
}
@Override
public int read() {
byte[] data = new byte[1];
int len = read(data, 0, 1);
return len <= 0 ? len : data[0] & 255;
}
@Override
public int read(byte[] b, int off, int len) {
if (pos >= size) {
return -1;
}
len = (int) Math.min(size - pos, len);
int x = seed, end = off + len;
// a fast and very simple pseudo-random number generator
// with a period length of 4 GB
// also good: x * 9 + 1, shift 6; x * 11 + 1, shift 7
while (off < end) {
x = (x << 4) + x + 1;
b[off++] = (byte) (x >> 8);
}
seed = x;
pos += len;
return len;
}
}
private void testDetectIllegalId() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
try {
store.length(new byte[]{3, 0, 0});
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
store.remove(new byte[]{3, 0, 0});
fail();
} catch (IllegalArgumentException e) {
// expected
}
map.put(0L, new byte[]{3, 0, 0});
InputStream in = store.get(new byte[]{2, 1, 0});
try {
in.read();
fail();
} catch (IllegalArgumentException e) {
// expected
}
}
private void testTreeStructure() throws IOException {
final AtomicInteger reads = new AtomicInteger();
Map<Long, byte[]> map = new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
@Override
public byte[] get(Object k) {
reads.incrementAndGet();
return super.get(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(100);
byte[] id = store.put(new ByteArrayInputStream(new byte[10000]));
InputStream in = store.get(id);
assertEquals(0, in.read(new byte[0]));
assertEquals(0, in.read());
assertEquals(3, reads.get());
}
private void testFormat() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(123);
byte[] id;
id = store.put(new ByteArrayInputStream(new byte[200]));
assertEquals(200, store.length(id));
assertEquals("02c8018801", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[0]));
assertEquals("", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[1]));
assertEquals("000100", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[3]));
assertEquals("0003000000", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[10]));
assertEquals("010a8901", StringUtils.convertBytesToHex(id));
byte[] combined = StringUtils.convertHexToBytes("0001aa0002bbcc");
assertEquals(3, store.length(combined));
InputStream in = store.get(combined);
assertEquals(1, in.skip(1));
assertEquals(0xbb, in.read());
assertEquals(1, in.skip(1));
}
private void testWithExistingData() throws IOException {
final AtomicInteger tests = new AtomicInteger();
Map<Long, byte[]> map = new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
@Override
public boolean containsKey(Object k) {
tests.incrementAndGet();
return super.containsKey(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(0);
for (int i = 0; i < 10; i++) {
store.put(new ByteArrayInputStream(new byte[20]));
}
assertEquals(10, map.size());
assertEquals(10, tests.get());
for (int i = 0; i < 10; i++) {
map.containsKey(i);
}
assertEquals(20, tests.get());
store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(0);
assertEquals(0, store.getNextKey());
for (int i = 0; i < 5; i++) {
store.put(new ByteArrayInputStream(new byte[20]));
}
assertEquals(88, tests.get());
assertEquals(15, store.getNextKey());
assertEquals(15, map.size());
for (int i = 0; i < 15; i++) {
map.containsKey(i);
}
}
private void testWithFullMap() throws IOException {
final AtomicInteger tests = new AtomicInteger();
Map<Long, byte[]> map = new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
@Override
public boolean containsKey(Object k) {
tests.incrementAndGet();
if (((Long) k) < Long.MAX_VALUE / 2) {
// simulate a *very* full map
return true;
}
return super.containsKey(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(20);
store.setMaxBlockSize(100);
store.setNextKey(0);
store.put(new ByteArrayInputStream(new byte[100]));
assertEquals(1, map.size());
assertEquals(64, tests.get());
assertEquals(Long.MAX_VALUE / 2 + 1, store.getNextKey());
}
private void testLoop() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
assertEquals(256 * 1024, store.getMaxBlockSize());
assertEquals(256, store.getMinBlockSize());
store.setNextKey(0);
assertEquals(0, store.getNextKey());
test(store, 10, 20, 1000);
for (int i = 0; i < 20; i++) {
test(store, 0, 128, i);
test(store, 10, 128, i);
}
for (int i = 20; i < 200; i += 10) {
test(store, 0, 128, i);
test(store, 10, 128, i);
}
}
private void test(StreamStore store, int minBlockSize, int maxBlockSize,
int length) throws IOException {
store.setMinBlockSize(minBlockSize);
assertEquals(minBlockSize, store.getMinBlockSize());
store.setMaxBlockSize(maxBlockSize);
assertEquals(maxBlockSize, store.getMaxBlockSize());
long next = store.getNextKey();
Random r = new Random(length);
byte[] data = new byte[length];
r.nextBytes(data);
byte[] id = store.put(new ByteArrayInputStream(data));
if (length > 0 && length >= minBlockSize) {
assertFalse(store.isInPlace(id));
} else {
assertTrue(store.isInPlace(id));
}
long next2 = store.getNextKey();
if (length > 0 && length >= minBlockSize) {
assertTrue(next2 > next);
} else {
assertEquals(next, next2);
}
if (length == 0) {
assertEquals(0, id.length);
}
assertEquals(length, store.length(id));
InputStream in = store.get(id);
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copy(in, out);
assertTrue(Arrays.equals(data, out.toByteArray()));
in = store.get(id);
in.close();
assertEquals(-1, in.read());
in = store.get(id);
assertEquals(0, in.skip(0));
if (length > 0) {
assertEquals(1, in.skip(1));
if (length > 1) {
assertEquals(data[1] & 255, in.read());
if (length > 2) {
assertEquals(1, in.skip(1));
if (length > 3) {
assertEquals(data[3] & 255, in.read());
}
} else {
assertEquals(0, in.skip(1));
}
} else {
assertEquals(-1, in.read());
}
} else {
assertEquals(0, in.skip(1));
}
if (length > 12) {
in = store.get(id);
assertEquals(12, in.skip(12));
assertEquals(data[12] & 255, in.read());
long skipped = 0;
while (true) {
long s = in.skip(Integer.MAX_VALUE);
if (s == 0) {
break;
}
skipped += s;
}
assertEquals(length - 13, skipped);
assertEquals(-1, in.read());
}
store.remove(id);
assertEquals(0, store.getMap().size());
}
}