/*
* This is a common dao with basic CRUD operations and is not limited to any
* persistent layer implementation
*
* Copyright (C) 2010 Imran M Yousuf (imyousuf@smartitengineering.com)
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
package com.smartitengineering.dao.hbase.autoincrement;
import com.google.inject.AbstractModule;
import com.smartitengineering.dao.hbase.ddl.HBaseTableConfiguration;
import com.smartitengineering.dao.hbase.ddl.HBaseTableGenerator;
import com.smartitengineering.dao.hbase.ddl.config.json.ConfigurationJsonParser;
import com.smartitengineering.util.bean.guice.GuiceUtil;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import junit.framework.Assert;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author imyousuf
*/
public class AutoIncrementRowIdForLongTest {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final Logger LOGGER = LoggerFactory.getLogger(AutoIncrementRowIdForLongTest.class);
private static final String TABLE_NAME = "test";
private static final String FAMILY_NAME = "family";
private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY_NAME);
private static final byte[] CELL_BYTES = Bytes.toBytes("cell");
private static final int PORT = 10080;
private static final int THREAD_COUNT = 100;
private static final String URI_FOR_TABLE = new StringBuilder("http://localhost:").append(PORT).append('/').append(
TABLE_NAME).toString();
private static Server jettyServer;
private static HttpClient httpClient;
private static final Set<Long> ids = new TreeSet<Long>();
private static HTablePool pool;
@BeforeClass
public static void globalSetup() throws Exception {
/*
* Start HBase and initialize tables
*/
//-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
System.setProperty("javax.xml.parsers.DocumentBuilderFactory",
"com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl");
try {
TEST_UTIL.startMiniCluster();
}
catch (Exception ex) {
LOGGER.error(ex.getMessage(), ex);
}
//Create table for testing
InputStream classpathResource = AutoIncrementRowIdForLongTest.class.getClassLoader().getResourceAsStream(
"ddl-config-sample1.json");
Collection<HBaseTableConfiguration> configs = ConfigurationJsonParser.getConfigurations(classpathResource);
try {
new HBaseTableGenerator(configs, TEST_UTIL.getConfiguration(), true).generateTables();
}
catch (Exception ex) {
LOGGER.error("Could not create table!", ex);
Assert.fail(ex.getMessage());
}
pool = new HTablePool(TEST_UTIL.getConfiguration(), 200);
//Start web app
jettyServer = new Server(PORT);
final String webapp = "./src/main/webapp/";
if (!new File(webapp).exists()) {
throw new IllegalStateException("WebApp file/dir does not exist!");
}
Properties properties = new Properties();
properties.setProperty(GuiceUtil.CONTEXT_NAME_PROP, "com.smartitengineering.dao.impl.hbase");
properties.setProperty(GuiceUtil.IGNORE_MISSING_DEP_PROP, Boolean.TRUE.toString());
properties.setProperty(GuiceUtil.MODULES_LIST_PROP, TestModule.class.getName());
GuiceUtil.getInstance(properties).register();
WebAppContext webAppHandler = new WebAppContext(webapp, "/");
jettyServer.setHandler(webAppHandler);
jettyServer.setSendDateHeader(true);
jettyServer.start();
//Initialize client
final MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager();
manager.getParams().setDefaultMaxConnectionsPerHost(THREAD_COUNT);
manager.getParams().setMaxTotalConnections(THREAD_COUNT);
httpClient = new HttpClient(manager);
}
public static void globalTeardown() throws Exception {
jettyServer.stop();
TEST_UTIL.shutdownMiniCluster();
}
@Test
public void testFirstPrimaryKey() throws IOException {
final long id = getId();
Assert.assertEquals(1, Long.MAX_VALUE - id);
final byte[] row = Bytes.toBytes(id);
Put put = new Put(row);
final byte[] toBytes = Bytes.toBytes("value");
put.add(FAMILY_BYTES, CELL_BYTES, toBytes);
HTableInterface table = pool.getTable(TABLE_NAME);
table.put(put);
Get get = new Get(row);
final Result get1 = table.get(get);
ids.add(Long.MAX_VALUE - id);
Assert.assertNotNull(get1);
Assert.assertTrue(Arrays.equals(row, get1.getRow()));
Assert.assertTrue(Arrays.equals(toBytes, get1.getValue(FAMILY_BYTES, CELL_BYTES)));
}
@Test
public void testConsequetive100Keys() throws IOException {
for (long id = 2; id < 102; ++id) {
final long rid = getId();
Assert.assertEquals(id, Long.MAX_VALUE - rid);
final byte[] row = Bytes.toBytes(rid);
Put put = new Put(row);
final byte[] toBytes = Bytes.toBytes("value " + id);
put.add(FAMILY_BYTES, CELL_BYTES, toBytes);
HTableInterface table = pool.getTable(TABLE_NAME);
table.put(put);
Get get = new Get(row);
final Result get1 = table.get(get);
ids.add(id);
Assert.assertNotNull(get1);
Assert.assertTrue(Arrays.equals(row, get1.getRow()));
Assert.assertTrue(Arrays.equals(toBytes, get1.getValue(FAMILY_BYTES, CELL_BYTES)));
}
}
@Test
public void testMultithreadedKeys() throws Exception {
ExecutorService service = Executors.newFixedThreadPool(THREAD_COUNT);
final long start = 102;
final int bound = THREAD_COUNT;
final int innerBound = 30;
List<Future> futures = new ArrayList<Future>();
final long startTime = System.currentTimeMillis();
for (int i = 0; i < bound; ++i) {
final int index = i;
futures.add(service.submit(new Runnable() {
@Override
public void run() {
for (int j = 0; j < innerBound; ++j) {
final HTableInterface table = pool.getTable(TABLE_NAME);
long id = index * bound + j + start;
final long id1 = getId();
synchronized (ids) {
final long mainId = Long.MAX_VALUE - id1;
Assert.assertFalse(ids.contains(mainId));
ids.add(mainId);
}
final byte[] row = Bytes.toBytes(id1);
Put put = new Put(row);
final byte[] toBytes = Bytes.toBytes("value " + id);
put.add(FAMILY_BYTES, CELL_BYTES, toBytes);
Result get1;
try {
table.put(put);
Get get = new Get(row);
get1 = table.get(get);
}
catch (Exception ex) {
LOGGER.error(ex.getMessage(), ex);
get1 = null;
Assert.fail(ex.getMessage());
}
pool.putTable(table);
Assert.assertNotNull(get1);
Assert.assertFalse(get1.isEmpty());
Assert.assertTrue(Arrays.equals(row, get1.getRow()));
Assert.assertTrue(Arrays.equals(toBytes, get1.getValue(FAMILY_BYTES, CELL_BYTES)));
}
}
}));
}
for (Future future : futures) {
future.get();
}
final long endTime = System.currentTimeMillis();
LOGGER.info("Time for " + (bound * innerBound) + " rows ID retrieval, put and get is " + (endTime - startTime) +
"ms");
Assert.assertEquals(bound * innerBound + start - 1, ids.size());
}
protected long getId() {
try {
PostMethod post = new PostMethod(URI_FOR_TABLE);
httpClient.executeMethod(post);
return Bytes.toLong(IOUtils.toByteArray(post.getResponseBodyAsStream()));
}
catch (Exception ex) {
throw new RuntimeException(ex);
}
}
@Test
public void testMultithreadedKeysSelfProvided() throws Exception {
ExecutorService service = Executors.newFixedThreadPool(THREAD_COUNT);
final long start = 3102;
final int bound = THREAD_COUNT;
final int innerBound = 30;
List<Future> futures = new ArrayList<Future>();
final long startTime = System.currentTimeMillis();
for (int i = 0; i < bound; ++i) {
final int index = i;
futures.add(service.submit(new Runnable() {
@Override
public void run() {
for (int j = 0; j < innerBound; ++j) {
final HTableInterface table = pool.getTable(TABLE_NAME);
long id = index * bound + j + start;
final long id1 = Long.MAX_VALUE - id;
synchronized (ids) {
final long mainId = Long.MAX_VALUE - id1;
Assert.assertFalse(ids.contains(mainId));
ids.add(mainId);
}
final byte[] row = Bytes.toBytes(id1);
Put put = new Put(row);
final byte[] toBytes = Bytes.toBytes("value " + id);
put.add(FAMILY_BYTES, CELL_BYTES, toBytes);
Result get1;
try {
table.put(put);
Get get = new Get(row);
get1 = table.get(get);
}
catch (Exception ex) {
LOGGER.error(ex.getMessage(), ex);
get1 = null;
Assert.fail(ex.getMessage());
}
pool.putTable(table);
Assert.assertNotNull(get1);
Assert.assertFalse(get1.isEmpty());
Assert.assertTrue(Arrays.equals(row, get1.getRow()));
Assert.assertTrue(Arrays.equals(toBytes, get1.getValue(FAMILY_BYTES, CELL_BYTES)));
}
}
}));
}
for (Future future : futures) {
future.get();
}
final long endTime = System.currentTimeMillis();
LOGGER.info("Time for " + (bound * innerBound) + " rows ID retrieval, put and get is " + (endTime - startTime) +
"ms");
Assert.assertEquals(bound * innerBound + start - 1, ids.size());
}
public static class TestModule extends AbstractModule {
@Override
protected void configure() {
bind(Configuration.class).toInstance(TEST_UTIL.getConfiguration());
}
}
}