/*-
* See the file LICENSE for redistribution information.
*
* Copyright (c) 1997, 2015 Oracle and/or its affiliates. All rights reserved.
*
* $Id$
*/
package db;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import com.sleepycat.db.*;
public class BulkExample {
private static final String dbFileName = "BulkExample.db";
private static final String pDbName = "primary";
private static final String sDbName = "secondary";
private static final String home = "BulkExample";
private static final int buffLen = 1024 * 1024;
/* Bulk methods demonstrated in the example */
private enum Operations {BULK_DELETE, BULK_READ, BULK_UPDATE}
private Database pdb;
private Environment env;
private Operations opt = Operations.BULK_READ;
private SecondaryDatabase sdb;
private int cachesize = 0, dups = 0, pagesize = 0;
private int num = 1000000;
private boolean secondary = false;
public static void main(String[] args) {
long startTime, endTime;
int count;
BulkExample app = new BulkExample();
/* Parse argument */
for (int i = 0; i < args.length; i++) {
if (args[i].compareTo("-c") == 0) {
if (i == args.length - 1)
usage();
i++;
app.cachesize = Integer.parseInt(args[i]);
System.out.println("[CONFIG] cachesize " + app.cachesize);
} else if (args[i].compareTo("-d") == 0) {
if (i == args.length - 1)
usage();
i++;
app.dups = Integer.parseInt(args[i]);
System.out.println(
"[CONFIG] number of duplicates " + app.dups);
} else if (args[i].compareTo("-n") == 0) {
if (i == args.length - 1)
usage();
i++;
app.num = Integer.parseInt(args[i]);
System.out.println("[CONFIG] number of keys " + app.num);
} else if (args[i].compareTo("-p") == 0) {
if (i == args.length - 1)
usage();
i++;
app.pagesize = Integer.parseInt(args[i]);
System.out.println("[CONFIG] pagesize " + app.pagesize);
} else if (args[i].compareTo("-D") == 0) {
app.opt = Operations.BULK_DELETE;
} else if (args[i].compareTo("-R") == 0) {
app.opt = Operations.BULK_READ;
} else if (args[i].compareTo("-S") == 0) {
app.secondary = true;
} else if (args[i].compareTo("-U") == 0) {
app.opt = Operations.BULK_UPDATE;
}else
usage();
}
/* Open environment and database(s) */
try {
/* If perform bulk update or delete, clean environment home */
app.cleanHome(app.opt != Operations.BULK_READ);
app.initDbs();
}catch (Exception e) {
e.printStackTrace();
app.closeDbs();
}
try {
/* Perform bulk read from existing primary db */
if (app.opt == Operations.BULK_READ) {
startTime = System.currentTimeMillis();
count = app.bulkRead();
endTime = System.currentTimeMillis();
app.printBulkOptStat(
Operations.BULK_READ, count, startTime, endTime);
} else {
/* Perform bulk update to populate the db */
startTime = System.currentTimeMillis();
app.bulkUpdate();
endTime = System.currentTimeMillis();
count = (app.dups == 0) ? app.num : app.num * app.dups;
app.printBulkOptStat(
Operations.BULK_UPDATE, count, startTime, endTime);
/* Perform bulk delete in primary db or secondary db */
if (app.opt == Operations.BULK_DELETE) {
startTime = System.currentTimeMillis();
if (app.secondary == false) {
/*
* Delete from the first key to a random one in
* primary db.
*/
count = new java.util.Random().nextInt(app.num);
count = app.bulkDelete(count);
} else {
/*
* Delete from the first key to a random one in
* secondary db.
*/
count = new java.util.Random().nextInt(
DataVal.tstring.length());
count = app.bulkSecondaryDelete(
(DataVal.tstring.getBytes())[count],
true);
}
endTime = System.currentTimeMillis();
app.printBulkOptStat(Operations.BULK_DELETE,
count, startTime, endTime);
}
}
} catch (DatabaseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
/* Close dbs */
app.closeDbs();
}
}
/* Perform bulk delete in primary db */
public int bulkDelete(int value) throws DatabaseException, IOException {
DatabaseEntry key = new DatabaseEntry();
Transaction txn = env.beginTransaction(null, null);
int count = 0;
try {
if (dups != 0) {
/* Delete a set of keys */
MultipleDataEntry keySet = new MultipleDataEntry();
keySet.setData(new byte[buffLen]);
keySet.setUserBuffer(buffLen, true);
for (int i = 0; i < value; i++) {
key.setData(intToByte(i));
keySet.append(key);
}
if (pdb.deleteMultiple(txn, keySet) !=
OperationStatus.SUCCESS)
count = 0;
else
count = value * dups;
} else {
/* Delete a set of key/value pairs */
MultipleKeyDataEntry pairSet = new MultipleKeyDataEntry();
pairSet.setData(new byte[buffLen * 2]);
pairSet.setUserBuffer(buffLen * 2, true);
DatabaseEntry data = new DatabaseEntry();
for (int i = 0; i < value; i++) {
key.setData(intToByte(i));
DataVal dataVal = new DataVal(i);
data.setData(dataVal.getBytes());
pairSet.append(key, data);
}
if (pdb.deleteMultipleKey(txn, pairSet) !=
OperationStatus.SUCCESS)
count = 0;
else
count = value;
}
txn.commit();
return count;
} catch (DatabaseException e) {
txn.abort();
throw e;
}
}
/* Perform bulk read in primary db */
public int bulkRead() throws DatabaseException {
ByteBuffer buffer1, buffer2;
Cursor cursor;
DatabaseEntry key, data;
MultipleKeyNIODataEntry pairs;
MultipleNIODataEntry keySet, dataSet;
int count;
key = new DatabaseEntry();
data = new DatabaseEntry();
count = 0;
Transaction txn = env.beginTransaction(null, null);
try {
cursor = pdb.openCursor(txn, null);
} catch (DatabaseException e) {
txn.abort();
return 0;
}
try {
if (dups == 0){
/* Get all records in a single key/data buffer */
pairs = new MultipleKeyNIODataEntry();
buffer1 = ByteBuffer.allocateDirect(buffLen * 2);
pairs.setDataNIO(buffer1);
pairs.setUserBuffer(buffLen * 2, true);
while (cursor.getNext(key, pairs, null) ==
OperationStatus.SUCCESS) {
while(pairs.next(key, data))
count++;
key = new DatabaseEntry();
}
} else {
/*
* Get all key/data pairs in two buffers, one for all keys,
* the other for all data.
*/
keySet = new MultipleNIODataEntry();
buffer1 = ByteBuffer.allocateDirect(buffLen);
keySet.setDataNIO(buffer1);
keySet.setUserBuffer(buffLen, true);
dataSet = new MultipleNIODataEntry();
buffer2 = ByteBuffer.allocateDirect(buffLen);
dataSet.setDataNIO(buffer2);
dataSet.setUserBuffer(buffLen, true);
while (cursor.getNext(keySet, dataSet, null) ==
OperationStatus.SUCCESS)
while(keySet.next(key) && dataSet.next(data))
count++;
}
cursor.close();
txn.commit();
return count;
} catch (DatabaseException e) {
cursor.close();
txn.abort();
throw e;
}
}
/* Perform bulk delete in secondary db */
public int bulkSecondaryDelete(byte value, boolean deletePair)
throws DatabaseException, IOException {
DatabaseEntry key = new DatabaseEntry();
byte[] valueBytes = new byte[1];
byte[] tstrBytes = DataVal.tstring.getBytes();
int i = 0;
int count = 0;
Transaction txn = env.beginTransaction(null, null);
try {
/* Prepare the buffer for a set of keys */
MultipleDataEntry keySet = new MultipleDataEntry();
keySet.setData(new byte[buffLen]);
keySet.setUserBuffer(buffLen, true);
/* Delete the given key and all keys prior to it */
do {
valueBytes[0] = tstrBytes[i];
key.setData(valueBytes);
keySet.append(key);
} while (value != tstrBytes[i++]);
if (sdb.deleteMultiple(txn, keySet) != OperationStatus.SUCCESS)
count = 0;
else
count = this.num / DataVal.tstring.length() * i;
if (i < this.num % DataVal.tstring.length())
count += i;
txn.commit();
return count;
} catch (DatabaseException e) {
txn.abort();
throw e;
}
}
/* Perform bulk update in primary db */
public void bulkUpdate() throws DatabaseException, IOException {
DatabaseEntry key, data;
MultipleDataEntry keySet, dataSet;
MultipleKeyDataEntry pairSet;
DataVal dataVal;
int i, j;
Transaction txn = env.beginTransaction(null, null);
key = new DatabaseEntry();
data = new DatabaseEntry();
try {
if (this.dups == 0) {
/*
* Bulk insert non-duplicate key/data pairs. Those pairs
* are filled into one single buffer.
*/
pairSet = new MultipleKeyDataEntry();
pairSet.setData(new byte[buffLen * 2]);
pairSet.setUserBuffer(buffLen * 2, true);
for (i = 0; i < this.num; i++) {
dataVal = new DataVal(i);
key.setData(intToByte(i));
data.setData(dataVal.getBytes());
pairSet.append(key, data);
}
pdb.putMultipleKey(txn, pairSet, true);
} else {
/* Bulk insert duplicate key/data pairs. All keys are
* filled into a buffer, and their data is filled into
* another buffer.
*/
keySet = new MultipleDataEntry();
keySet.setData(new byte[buffLen]);
keySet.setUserBuffer(buffLen, true);
dataSet = new MultipleDataEntry();
dataSet.setData(new byte[buffLen]);
dataSet.setUserBuffer(buffLen, true);
for (i = 0; i < this.num; i++) {
j = 0;
dataVal = new DataVal(i);
do {
key.setData(intToByte(i));
keySet.append(key);
dataVal.setDataId(j);
data.setData(dataVal.getBytes());
dataSet.append(data);
} while(++j < this.dups);
}
pdb.putMultiple(txn, keySet, dataSet, true);
}
txn.commit();
}catch (DatabaseException e1) {
txn.abort();
throw e1;
}catch (IOException e2) {
txn.abort();
throw e2;
}
}
/* Clean the home directory */
public void cleanHome(boolean clean) {
File file = new File(home);
/*
* If the home directory doesn't exist, create the directory.
* Unless, if it is requested to be cleaned, delete all and
* create a new one.
*/
if (!file.exists())
file.mkdir();
else if (clean == true) {
File[] files = file.listFiles();
for(int i=0; i< files.length; i++) {
files[i].delete();
}
file.delete();
file.mkdir();
}
}
/* Close database(s) and environment */
public void closeDbs() {
try {
if (this.secondary)
sdb.close();
pdb.close();
env.close();
} catch (DatabaseException e) {
System.exit(1);
}
}
/* Print statistics in bulk operations */
public void printBulkOptStat(
Operations opt, int count, long startTime, long endTime) {
System.out.print("[STAT] ");
if (opt == Operations.BULK_DELETE && !secondary)
System.out.print("Bulk delete ");
else if (opt == Operations.BULK_DELETE && secondary)
System.out.print("Bulk secondary delete ");
else if (opt == Operations.BULK_READ)
System.out.print("Bulk read ");
else
System.out.print("Bulk update ");
System.out.print(count + " records");
System.out.print(" in " + (endTime - startTime) + " ms");
System.out.println(", that is " +
(int)((double)1000 / (endTime - startTime) * (double)count) +
" records/second");
}
/* Initialize environment and database (s) */
public void initDbs() throws DatabaseException, FileNotFoundException {
/* Open transactional environment */
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true);
envConfig.setInitializeCache(true);
envConfig.setInitializeLocking(true);
envConfig.setInitializeLogging(true);
envConfig.setTransactional(true);
File envFile = new File(home);
try {
env = new Environment(envFile, envConfig);
} catch (Exception e) {
System.exit(1);
}
Transaction txn = env.beginTransaction(null, null);
try {
/* Open primary db in transaction */
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
dbConfig.setCacheSize(this.cachesize);
dbConfig.setPageSize(this.pagesize);
dbConfig.setType(DatabaseType.BTREE);
if (this.dups != 0)
dbConfig.setUnsortedDuplicates(true);
pdb = env.openDatabase(txn, dbFileName, pDbName, dbConfig);
/* Open secondary db in transaction */
if (this.secondary) {
SecondaryConfig sdbConfig = new SecondaryConfig();
sdbConfig.setTransactional(true);
sdbConfig.setAllowCreate(true);
sdbConfig.setCacheSize(this.cachesize);
sdbConfig.setPageSize(this.pagesize);
sdbConfig.setType(DatabaseType.BTREE);
sdbConfig.setKeyCreator(new FirstStrKeyCreator());
sdbConfig.setSortedDuplicates(true);
sdb = env.openSecondaryDatabase(
txn, dbFileName, sDbName, pdb, sdbConfig);
}
txn.commit();
} catch (DatabaseException e1) {
txn.abort();
throw e1;
} catch (FileNotFoundException e2) {
txn.abort();
throw e2;
}
}
/* Convert integer to byte array */
public byte[] intToByte(int i) {
byte[] dword = new byte[4];
dword[0] = (byte) (i & 0x00FF);
dword[1] = (byte) ((i >> 8) & 0x000000FF);
dword[2] = (byte) ((i >> 16) & 0x000000FF);
dword[3] = (byte) ((i >> 24) & 0x000000FF);
return dword;
}
/* Usage of BulkExample */
private static void usage() {
System.out.println("Usage: BulkExample \n" +
" -c cachesize [1000 * pagesize] \n" +
" -d number of duplicates [none] \n" +
" -n number of keys [1000000] \n" +
" -p pagesize [65536] \n" +
" -D perform bulk delete \n" +
" -R perform bulk read \n" +
" -S perform bulk read in secondary database \n" +
" -U perform bulk update \n");
System.exit(1);
}
class DataVal {
/* String template */
public static final String tstring = "0123456789abcde";
private String dataString;
private int dataId;
/*
* Construct DataVal with given kid and id. The dataString is the
* rotated tstring from the kid position. The dataId is the id.
*/
public DataVal(int kid) {
int rp = kid % tstring.length();
if (rp == 0)
this.dataString = tstring;
else
this.dataString = tstring.substring(rp) +
tstring.substring(0, rp - 1);
this.dataId = 0;
}
/* Parse the object including dataId and dataString to byte array. */
public byte[] getBytes() throws IOException {
int i;
byte[] strBytes = dataString.getBytes();
byte[] intBytes = intToByte(dataId);
byte[] bytes = new byte[4 + strBytes.length];
for (i = 0; i < 4 + strBytes.length; i++) {
if (i < 4)
bytes[i] = intBytes[i];
else
bytes[i] = strBytes[i - 4];
}
return bytes;
}
public int getDataId() {
return dataId;
}
public void setDataId(int id) {
dataId = id;
}
public String getDataString() {
return dataString;
}
}
/* Secondary key creator */
class FirstStrKeyCreator implements SecondaryKeyCreator {
public boolean createSecondaryKey(SecondaryDatabase secondary,
DatabaseEntry key, DatabaseEntry data, DatabaseEntry result)
throws DatabaseException {
/*
* Create the secondary key. It would be the first character in
* dataString of DataVal, that is the 5th byte in data.
*/
byte[] firstStr = new byte[1];
firstStr[0] = (data.getData())[4];
result.setData(firstStr);
result.setSize(1);
return true;
}
}
}