package com.alibaba.doris.dataserver.store.log.db;
import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import junit.framework.TestCase;
import org.junit.Test;
import com.alibaba.doris.common.data.ActionPair;
import com.alibaba.doris.common.data.Key;
import com.alibaba.doris.common.data.Pair;
import com.alibaba.doris.common.data.Value;
import com.alibaba.doris.common.data.impl.KeyImpl;
import com.alibaba.doris.common.data.impl.ValueImpl;
import com.alibaba.doris.common.data.util.ByteUtils;
import com.alibaba.doris.common.router.virtual.VirtualRouterImpl;
import com.alibaba.doris.dataserver.store.Storage;
import com.alibaba.doris.dataserver.store.StorageConfig;
import com.alibaba.doris.dataserver.store.log.LogStorageDriver;
/**
* @author ajun Email:jack.yuj@alibaba-inc.com
*/
public class LogStoragePressureTest extends TestCase {
@Test
public void testlogSroage() throws InterruptedException {
LogStorageDriver driver = new LogStorageDriver();
StorageConfig config = new StorageConfig();
config.setPropertiesFile("log_storage_presure_test.properties");
driver.init(config);
ClumpConfigure clumpConfig = driver.getClumpConfigure();
clumpConfig.setMaxFileSize(1024 * 1024 * 100);
Storage storage = driver.createStorage();
try {
storage.open();
runTask(storage);
storage.close();
storage.open();
testRead(storage);
storage.close();
storage.open();
testDelete(storage);
storage.close();
} finally {
storage.close();
}
assertTrue("断言压力测试写入读取数据校验失败", errCount.get() == 0);
}
public static void testRead(Storage storage) {
int count = 0, errorCount = 0;
long start = System.currentTimeMillis();
try {
Iterator<Pair> iterator = storage.iterator();
while (iterator.hasNext()) {
Pair p = iterator.next();
String key = p.getKey().getKey();
if (p != null && p.getKey() != null) {
if (p instanceof ActionPair) {
if (((ActionPair) p).getActionType() == ActionPair.Type.SET) {
Value value = p.getValue();
String v = ByteUtils.byteToString(value.getValueBytes());
if (v.startsWith(key)) {
count++;
continue;
} else {
errorCount++;
System.out.println("count=" + count + "key=" + key + " error value:" + v);
continue;
}
}
}
count++;
} else {
errorCount++;
}
}
} catch (Throwable e) {
System.out.println(" c:" + count + " error:" + errorCount);
e.printStackTrace();
}
long time = System.currentTimeMillis() - start;
if (time <= 0) {
time = 1;
}
System.out.println("time(ms):" + time + " c:" + count + " error:" + errorCount + " avg:" + (time)
/ (count + errorCount));
}
private static void testDelete(Storage storage) {
int[] vnodeList = ((VirtualRouterImpl) VirtualRouterImpl.getInstance()).getVirtualNode();
ArrayList<Integer> list = new ArrayList<Integer>(vnodeList.length);
for (int i = 0; i < vnodeList.length; i++) {
list.add(vnodeList[i]);
}
assertTrue(storage.delete(list));
try {
Iterator<Pair> iterator = storage.iterator();
assertFalse(iterator.hasNext());
} catch (Exception e) {
fail(e.getLocalizedMessage());
}
}
private static void runTask(final Storage storage) throws InterruptedException {
final int len = 10000;
int nThreads = 10;
StringBuilder value = new StringBuilder();
for (int i = 0; i < 512; i++) {
value.append("1");
}
ExecutorService executor = Executors.newFixedThreadPool(nThreads);
final CountDownLatch startGate = new CountDownLatch(1);
final CountDownLatch endGate = new CountDownLatch(nThreads);
for (int i = 0; i < nThreads; i++) {
executor.execute(new Runnable() {
public void run() {
try {
Random random = new Random();
String key = Thread.currentThread().getName() + "_KEY_";
startGate.await();
for (int i = 0; i < len; i++) {
StringBuilder value = new StringBuilder();
int strLen = random.nextInt(10240);
for (int j = 0; j < strLen; j++) {
value.append("1");
}
String k_ = key + i;
String v = k_ + value.toString();
sizeCount.addAndGet(value.length());
Key keyObj = new KeyImpl(101, k_, 0);
Value valueObj = new ValueImpl(ByteUtils.stringToByte(v), System.currentTimeMillis());
storage.set(keyObj, valueObj);
storage.delete(keyObj);
}
} catch (Exception e) {
fail(e.getLocalizedMessage());
} finally {
endGate.countDown();
}
}
});
}
Thread.sleep(1000);
long start = System.currentTimeMillis();
startGate.countDown();
endGate.await();
long end = System.currentTimeMillis();
executor.shutdown();
long size = (sizeCount.get() * 2) / 1024;
if (size <= 0) size = 1;
long total = end - start;
if (total <= 0) total = 1;
Thread.sleep(100);
System.out.println("-------------------" + storage.getClass().getName() + "---------------");
System.out.println("total time:" + total + " error count: " + errCount.get());
System.out.println("size:" + size + "KB" + " total:" + total);
System.out.println("len=" + len * nThreads * 2 + " avg:" /* + total / len */+ " size:" + size + "KB " /*
* +
* (size
* ) /
* ((
* total
* ) /
* 1000)
*/
+ "K/S total:" + total);
System.out.println("-------------------end---------------");
}
protected static ClumpConfigure getClumpConfigure() {
ClumpConfigure config = new ClumpConfigure();
config.setPath(LogStoragePressureTest.class.getClassLoader().getResource("").getPath() + "test_data"
+ File.separatorChar);
config.setReadBufferSize(1024 * 512);
config.setWriteBufferSize(1024 * 512);
config.setWriteDirect(true);// 不缓存数据直接写入磁盘
return config;
}
private static AtomicLong sizeCount = new AtomicLong();
private static AtomicInteger errCount = new AtomicInteger();
}