/*
* Copyright (C) 2012, 2016 higherfrequencytrading.com
* Copyright (C) 2016 Roman Leventov
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map;
import org.junit.Ignore;
import org.junit.Test;
import java.beans.XMLEncoder;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
import static org.junit.Assert.*;
/**
* Created by peter.lawrey on 06/12/14.
*/
public class LargeEntriesTest {
@Test
public void testLargeStrings() throws ExecutionException, InterruptedException, IOException {
final int ENTRIES = 250;
final int ENTRY_SIZE = 100 * 1024;
File file = File.createTempFile("largeEntries" + System.currentTimeMillis(), ".deleteme");
file.deleteOnExit();
try (final ChronicleMap<String, String> map = ChronicleMapBuilder
.of(String.class, String.class)
// .valueReaderAndDataAccess(, SnappyStringMarshaller.INSTANCE, )
.actualSegments(1) // to force an error.
.entries(ENTRIES)
.averageKeySize(10)
.averageValueSize(ENTRY_SIZE)
.putReturnsNull(true)
.createPersistedTo(file)) {
warmUpCompression(ENTRY_SIZE);
int threads = 4; //Runtime.getRuntime().availableProcessors();
ExecutorService es = Executors.newFixedThreadPool(threads);
final int block = ENTRIES / threads;
for (int i = 0; i < 3; i++) {
long start = System.currentTimeMillis();
List<Future<?>> futureList = new ArrayList<>();
for (int t = 0; t < threads; t++) {
final int finalT = t;
futureList.add(es.submit(new Runnable() {
@Override
public void run() {
exerciseLargeStrings(map, finalT * block, finalT * block + block,
ENTRY_SIZE);
}
}));
}
for (Future<?> future : futureList) {
future.get();
}
long time = System.currentTimeMillis() - start;
long operations = 3;
System.out.printf("Put/Get rate was %.1f MB/s%n",
operations * ENTRIES * ENTRY_SIZE / 1e6 / (time / 1e3));
}
es.shutdown();
es.awaitTermination(1, TimeUnit.MINUTES);
assertTrue(es.isTerminated());
}
file.delete();
}
private void warmUpCompression(int entrySize) {
// String value = generateValue(entrySize);
// DirectBytes bytes = DirectStore.allocate(entrySize / 6).bytes();
// for (int i = 0; i < 5; i++) {
// // warmup to compression.
// bytes.clear();
// SnappyStringMarshaller.INSTANCE.write(bytes, value);
// bytes.flip();
// SnappyStringMarshaller.INSTANCE.read(bytes);
// }
// bytes.release();
}
@Test
@Ignore("Performance Test")
public void testLargeStringsPerf() throws ExecutionException, InterruptedException, IOException {
doLargeEntryPerf(10000, 100 * 1024);
doLargeEntryPerf(1000000, 1024);
doLargeEntryPerf(100000, 10 * 1024);
doLargeEntryPerf(10000, 100 * 1024);
doLargeEntryPerf(3000, 1024 * 1024);
}
private void doLargeEntryPerf(int ENTRIES, final int ENTRY_SIZE) throws IOException, InterruptedException, ExecutionException {
System.out.printf("Testing %,d entries of %,d KB%n", ENTRIES, ENTRY_SIZE / 1024);
File file = File.createTempFile("largeEntries", ".deleteme");
file.deleteOnExit();
final ChronicleMap<String, String> map = ChronicleMapBuilder
.of(String.class, String.class)
// .valueReaderAndDataAccess(, SnappyStringMarshaller.INSTANCE, )
.entries(ENTRIES)
.averageKeySize(10)
.averageValueSize(ENTRY_SIZE)
.putReturnsNull(true)
.createPersistedTo(file);
{
// warmUpCompression(ENTRY_SIZE);
int threads = Runtime.getRuntime().availableProcessors();
ExecutorService es = Executors.newFixedThreadPool(threads);
final int block = ENTRIES / threads;
for (int i = 0; i < 3; i++) {
long start = System.currentTimeMillis();
List<Future<?>> futureList = new ArrayList<>();
for (int t = 0; t < threads; t++) {
final int finalT = t;
futureList.add(es.submit(new Runnable() {
@Override
public void run() {
exerciseLargeStrings(map, finalT * block, finalT * block + block,
ENTRY_SIZE);
}
}));
}
for (Future<?> future : futureList) {
future.get();
}
long time = System.currentTimeMillis() - start;
long operations = 3;
System.out.printf("Put/Get rate was %.1f MB/s%n", operations * ENTRIES * ENTRY_SIZE / 1e6 / (time / 1e3));
}
es.shutdown();
if (es.isTerminated())
map.close();
}
}
void exerciseLargeStrings(ChronicleMap<String, String> map, int start, int finish, int entrySize) {
/*
final Thread thisThread = Thread.currentThread();
Thread monitor = new Thread(new Runnable() {
@Override
public void run() {
StringBuilder sb = new StringBuilder();
sb.append(thisThread).append("\n");
try {
while (!Thread.currentThread().isInterrupted()) {
Jvm.pause(50);
StackTraceElement[] stackTrace = thisThread.getStackTrace();
for (int i = 0; i < 4 && i < stackTrace.length; i++)
sb.append("\tat ").append(stackTrace[i]).append("\n");
sb.append("\n");
}
} catch (InterruptedException e) {
}
System.out.println(sb);
}
});
monitor.start();
*/
String value = generateValue(entrySize);
for (int i = start; i < finish; i++) {
String key = "key-" + i;
String object;
map.put(key, value);
object = map.get(key);
assertTrue(key, map.containsKey(key));
assertNotNull(key, object);
assertEquals(key, entrySize, object.length());
}
// monitor.interrupt();
for (int i = start; i < finish; i++) {
// System.out.println(i);
String key = "key-" + i;
String object = map.get(key);
assertNotNull(key, object);
assertEquals(key, entrySize, object.length());
}
}
private String generateValue(int entrySize) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
XMLEncoder xml = new XMLEncoder(baos);
Map<String, String> map2 = new HashMap<>();
Random rand = new Random(1);
for (int i = 0; i < entrySize / 80; i++)
map2.put("key-" + i, "value-" + rand.nextInt(1000));
xml.writeObject(map2);
xml.close();
return baos.toString().substring(0, entrySize);
}
}