/*
* This is a common dao with basic CRUD operations and is not limited to any
* persistent layer implementation
*
* Copyright (C) 2010 Imran M Yousuf (imyousuf@smartitengineering.com)
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
package com.smartitengineering.dao.impl.hbase;
import com.google.inject.Inject;
import com.google.inject.internal.Nullable;
import com.google.inject.name.Named;
import com.smartitengineering.dao.common.queryparam.BasicCompoundQueryParameter;
import com.smartitengineering.dao.common.queryparam.BiOperandQueryParameter;
import com.smartitengineering.dao.common.queryparam.CompositionQueryParameter;
import com.smartitengineering.dao.common.queryparam.MatchMode;
import com.smartitengineering.dao.common.queryparam.OperatorType;
import com.smartitengineering.dao.common.queryparam.ParameterType;
import com.smartitengineering.dao.common.queryparam.QueryParameter;
import com.smartitengineering.dao.common.queryparam.QueryParameterCastHelper;
import com.smartitengineering.dao.common.queryparam.QueryParameterWithOperator;
import com.smartitengineering.dao.common.queryparam.QueryParameterWithPropertyName;
import com.smartitengineering.dao.common.queryparam.QueryParameterWithValue;
import com.smartitengineering.dao.common.queryparam.ValueOnlyQueryParameter;
import com.smartitengineering.dao.impl.hbase.spi.AsyncExecutorService;
import com.smartitengineering.dao.impl.hbase.spi.Callback;
import com.smartitengineering.dao.impl.hbase.spi.FilterConfig;
import com.smartitengineering.dao.impl.hbase.spi.LockAttainer;
import com.smartitengineering.dao.impl.hbase.spi.LockType;
import com.smartitengineering.dao.impl.hbase.spi.MergeService;
import com.smartitengineering.dao.impl.hbase.spi.ObjectRowConverter;
import com.smartitengineering.dao.impl.hbase.spi.SchemaInfoProvider;
import com.smartitengineering.dao.impl.hbase.spi.impl.DiffBasedMergeService;
import com.smartitengineering.dao.impl.hbase.spi.impl.RangeComparator;
import com.smartitengineering.domain.PersistentDTO;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SkipFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A common DAO implementation for HBase. Please note that all parameters for reading (i.e. Scan) assumes that the
* toString() method returns the string representation of the value to be compared in byte[] form.
* @author imyousuf
*/
public class CommonDao<Template extends PersistentDTO<? extends PersistentDTO, ? extends Comparable, ? extends Long>, IdType extends Serializable>
implements
com.smartitengineering.dao.common.CommonDao<Template, IdType> {
public static final int DEFAULT_MAX_ROWS = 1000;
@Inject
private ObjectRowConverter<Template> converter;
@Inject
private SchemaInfoProvider<Template, IdType> infoProvider;
@Inject
private AsyncExecutorService executorService;
@Inject
private ExecutorService resultExecutorService;
@Inject
@Named("mergeEnabled")
private Boolean mergeEnabled = false;
@Inject
@Nullable
private MergeService<Template, IdType> mergeService;
@Inject
private LockAttainer<Template, IdType> lockAttainer;
@Inject(optional = true)
private LockType lockType;
private final String errorMessageFormat = "Operation of row %s from table %s has failed optimistically!";
private int maxRows = -1;
protected final Logger logger = LoggerFactory.getLogger(getClass());
public AsyncExecutorService getExecutorService() {
return executorService;
}
public void setExecutorService(AsyncExecutorService executorService) {
this.executorService = executorService;
}
public int getMaxRows() {
return maxRows;
}
@Inject
public void setMaxRows(@Named("maxRows") Integer maxRows) {
this.maxRows = maxRows;
}
public ObjectRowConverter<Template> getConverter() {
return converter;
}
public void setConverter(ObjectRowConverter<Template> converter) {
this.converter = converter;
}
public SchemaInfoProvider<Template, IdType> getInfoProvider() {
return infoProvider;
}
public void setInfoProvider(SchemaInfoProvider<Template, IdType> infoProvider) {
this.infoProvider = infoProvider;
}
protected String getDefaultTableName() {
return getInfoProvider().getMainTableName();
}
protected int getMaxScanRows() {
return getMaxRows() > 0 ? getMaxRows() : DEFAULT_MAX_ROWS;
}
protected int getMaxScanRows(List<QueryParameter> params) {
if (params != null && !params.isEmpty()) {
for (QueryParameter param : params) {
if (ParameterType.PARAMETER_TYPE_MAX_RESULT.equals(param.getParameterType())) {
ValueOnlyQueryParameter<Integer> queryParameter = QueryParameterCastHelper.VALUE_PARAM_HELPER.cast(param);
return queryParameter.getValue();
}
}
}
return getMaxScanRows();
}
/*
* READ OPERATIONS
*/
/*
* Unsupported read operations
*/
@Override
public Set<Template> getAll() {
logger.info("Get ALL Executed!");
return executorService.execute(getDefaultTableName(), new Callback<Set<Template>>() {
@Override
public Set<Template> call(HTableInterface tableInterface) throws Exception {
final Scan scan = new Scan();
RowFilter rowFilter = new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(new byte[0]));
scan.setFilter(rowFilter);
final int maxRows = getMaxScanRows();
return new LinkedHashSet<Template>(CommonDao.this.scanList(tableInterface, scan, maxRows));
}
});
}
@Override
public <OtherTemplate extends Object> OtherTemplate getOther(final List<QueryParameter> query) {
return executorService.execute(getDefaultTableName(), new Callback<OtherTemplate>() {
@Override
public OtherTemplate call(HTableInterface tableInterface) throws Exception {
ResultScanner scanner = tableInterface.getScanner(formScan(query));
try {
Result result = scanner.next();
if (result == null || result.isEmpty()) {
return null;
}
else {
return (OtherTemplate) result.getNoVersionMap();
}
}
finally {
if (scanner != null) {
scanner.close();
}
}
}
});
}
@Override
public <OtherTemplate> List<OtherTemplate> getOtherList(final List<QueryParameter> query) {
return executorService.execute(getDefaultTableName(), new Callback<List<OtherTemplate>>() {
@Override
public List<OtherTemplate> call(HTableInterface tableInterface) throws Exception {
ResultScanner scanner = tableInterface.getScanner(formScan(query));
try {
Result[] results = scanner.next(getMaxScanRows(query));
if (results == null) {
return Collections.emptyList();
}
else {
final ArrayList<OtherTemplate> templates = new ArrayList<OtherTemplate>(results.length);
for (final Result result : results) {
if (result == null || result.isEmpty()) {
continue;
}
else {
templates.add((OtherTemplate) result.getNoVersionMap());
}
}
return templates;
}
}
finally {
if (scanner != null) {
scanner.close();
}
}
}
});
}
/*
* Supported read operations
*/
@Override
public Set<Template> getByIds(List<IdType> ids) {
LinkedHashSet<Future<Template>> set = new LinkedHashSet<Future<Template>>(ids.size());
LinkedHashSet<Template> resultSet = new LinkedHashSet<Template>(ids.size());
for (IdType id : ids) {
set.add(executorService.executeAsynchronously(getDefaultTableName(), getByIdCallback(id)));
}
for (Future<Template> future : set) {
try {
final Template get = future.get();
if (get != null) {
resultSet.add(get);
}
}
catch (Exception ex) {
logger.warn("Could not retrieve from Future...", ex);
}
}
return resultSet;
}
@Override
public Template getById(final IdType id) {
return executorService.execute(getDefaultTableName(), getByIdCallback(id));
}
protected Callback<Template> getByIdCallback(final IdType id) {
return new Callback<Template>() {
@Override
public Template call(HTableInterface tableInterface) throws Exception {
final byte[] rowId = getInfoProvider().getRowIdFromId(id);
Get get = new Get(rowId);
Result result = tableInterface.get(get);
if (result == null || result.isEmpty()) {
return null;
}
else {
return getConverter().rowsToObject(result, executorService);
}
}
};
}
@Override
public Template getSingle(final List<QueryParameter> query) {
return executorService.execute(getDefaultTableName(), new Callback<Template>() {
@Override
public Template call(HTableInterface tableInterface) throws Exception {
ResultScanner scanner = tableInterface.getScanner(formScan(query));
try {
Result result = scanner.next();
if (result == null || result.isEmpty()) {
return null;
}
else {
return getConverter().rowsToObject(result, executorService);
}
}
finally {
if (scanner != null) {
scanner.close();
}
}
}
});
}
@Override
public List<Template> getList(final List<QueryParameter> query) {
return executorService.execute(getDefaultTableName(), new Callback<List<Template>>() {
@Override
public List<Template> call(HTableInterface tableInterface) throws Exception {
final Scan scan = formScan(query);
final int maxRows = getMaxScanRows(query);
return CommonDao.this.scanList(tableInterface, scan, maxRows);
}
});
}
protected LockType getLockType() {
if (lockType == null) {
return LockType.getDefault();
}
return lockType;
}
protected List<Template> scanList(HTableInterface tableInterface, Scan scan, int maxRows) throws IOException,
InterruptedException,
ExecutionException {
logger.info("Scanning a list");
if (logger.isDebugEnabled()) {
logger.debug("Scanning for rows " + maxRows);
}
ResultScanner scanner = tableInterface.getScanner(scan);
try {
Result[] results = scanner.next(maxRows);
if (logger.isDebugEnabled()) {
logger.debug("ResultScanner " + scanner);
logger.debug("Results " + results);
if (results != null) {
logger.debug("Results " + results.length);
}
}
if (results == null) {
return Collections.emptyList();
}
else {
final ArrayList<Template> templates = new ArrayList<Template>(results.length);
ArrayList<Future<Template>> futureTemplates = new ArrayList<Future<Template>>(results.length);
for (final Result result : results) {
if (result == null || result.isEmpty()) {
logger.debug("Result is null or empty " + result + " " + (result != null ? result.isEmpty() : "TRUE"));
continue;
}
else {
futureTemplates.add(resultExecutorService.submit(new Callable<Template>() {
@Override
public Template call() throws Exception {
logger.debug("Converting row to object " + result);
return getConverter().rowsToObject(result, executorService);
}
}));
}
}
for (Future<Template> future : futureTemplates) {
Template template = future.get();
if (template != null) {
templates.add(template);
}
}
return templates;
}
}
catch (IOException ex) {
logger.warn(ex.getMessage(), ex);
throw ex;
}
catch (InterruptedException ex) {
logger.warn(ex.getMessage(), ex);
throw ex;
}
catch (ExecutionException ex) {
logger.warn(ex.getMessage(), ex);
throw ex;
}
finally {
if (scanner != null) {
scanner.close();
}
}
}
protected Scan formScan(List<QueryParameter> query) {
Scan scan = new Scan();
final Filter filter = getFilter(query, scan);
if (filter != null) {
scan.setFilter(filter);
}
scan.setCaching(getMaxScanRows(query));
return scan;
}
protected Filter getFilter(Collection<QueryParameter> queryParams, Scan scan) {
return getFilter(queryParams, scan, Operator.MUST_PASS_ALL);
}
protected Filter getFilter(Collection<QueryParameter> queryParams, Scan scan, Operator operator) {
return getFilter("", queryParams, scan, operator);
}
protected Filter getFilter(String namePrefix, Collection<QueryParameter> queryParams, Scan scan, Operator operator) {
final Filter filter;
if (queryParams != null && !queryParams.isEmpty()) {
List<Filter> filters = new ArrayList<Filter>(queryParams.size());
for (QueryParameter param : queryParams) {
switch (param.getParameterType()) {
case PARAMETER_TYPE_CONJUNCTION: {
BasicCompoundQueryParameter queryParameter =
QueryParameterCastHelper.BASIC_COMPOUND_PARAM_HELPER.cast(param);
Collection<QueryParameter> nestedParameters = queryParameter.getNestedParameters();
filters.add(getFilter(nestedParameters, scan, Operator.MUST_PASS_ALL));
break;
}
case PARAMETER_TYPE_NESTED_PROPERTY: {
CompositionQueryParameter queryParameter = QueryParameterCastHelper.COMPOSITION_PARAM_FOR_NESTED_TYPE.cast(
param);
Collection<QueryParameter> nestedParameters = queryParameter.getNestedParameters();
filters.add(getFilter(getPropertyName(namePrefix, param), nestedParameters, scan, Operator.MUST_PASS_ALL));
break;
}
case PARAMETER_TYPE_DISJUNCTION: {
BasicCompoundQueryParameter queryParameter =
QueryParameterCastHelper.BASIC_COMPOUND_PARAM_HELPER.cast(param);
Collection<QueryParameter> nestedParameters = queryParameter.getNestedParameters();
filters.add(getFilter(nestedParameters, scan, Operator.MUST_PASS_ONE));
break;
}
case PARAMETER_TYPE_PROPERTY: {
handlePropertyParam(namePrefix, param, filters);
break;
}
case PARAMETER_TYPE_FIRST_RESULT: {
Object value = getValue(param);
scan.setStartRow(Bytes.toBytes(value.toString()));
break;
}
case PARAMETER_TYPE_UNIT_PROP: {
final String propertyName = getPropertyName(namePrefix, param);
final String configPropertyName;
final int indexOfColon = propertyName.indexOf(":");
final String dynaPropName;
if (indexOfColon > -1) {
configPropertyName = propertyName.substring(0, indexOfColon);
dynaPropName = propertyName.substring(indexOfColon + 1);
}
else {
configPropertyName = propertyName;
dynaPropName = null;
}
FilterConfig config = getInfoProvider().getFilterConfig(configPropertyName);
if (config != null) {
if (StringUtils.isNotBlank(dynaPropName)) {
scan.addColumn(config.getColumnFamily(), Bytes.toBytes(dynaPropName));
}
else if (config.getColumnQualifier() != null && config.getColumnQualifier().length > 0) {
scan.addColumn(config.getColumnFamily(), config.getColumnQualifier());
}
else {
scan.addFamily(config.getColumnFamily());
}
}
break;
}
default:
//Do nothing
}
}
if (!filters.isEmpty()) {
FilterList filterList = new FilterList(operator, filters);
filter = filterList;
}
else {
filter = null;
}
}
else {
filter = new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(new byte[0]));
}
return filter;
}
protected void handlePropertyParam(String namePrefix, QueryParameter queryParameter, List<Filter> filters) {
OperatorType operator = getOperator(queryParameter);
Object parameter = getValue(queryParameter);
final boolean byteArray;
final boolean isFilter;
final byte[] paramAsArray;
if (logger.isInfoEnabled()) {
logger.info("Class of parameter is " + parameter.getClass());
}
if (parameter instanceof byte[]) {
paramAsArray = (byte[]) parameter;
byteArray = true;
}
else {
paramAsArray = null;
byteArray = false;
}
FilterConfig filterConfig = getInfoProvider().getFilterConfig(getPropertyName(namePrefix, queryParameter));
switch (operator) {
case OPERATOR_EQUAL: {
filters.add(getCellFilter(filterConfig, CompareOp.EQUAL, !byteArray ? Bytes.toBytes(parameter.toString()) :
paramAsArray, parameter));
return;
}
case OPERATOR_LESSER: {
logger.info("Lesser operator. Is with byte array - " + byteArray);
filters.add(getCellFilter(filterConfig, CompareOp.LESS, !byteArray ? Bytes.toBytes(parameter.toString()) :
paramAsArray, parameter));
return;
}
case OPERATOR_LESSER_EQUAL: {
logger.info("Lesser than equal to operator. Is with byte array - " + byteArray);
filters.add(getCellFilter(filterConfig, CompareOp.LESS_OR_EQUAL, !byteArray ?
Bytes.toBytes(parameter.toString()) : paramAsArray, parameter));
return;
}
case OPERATOR_GREATER: {
logger.info("Greater operator. Is with byte array - " + byteArray);
filters.add(getCellFilter(filterConfig, CompareOp.GREATER, !byteArray ? Bytes.toBytes(parameter.toString()) :
paramAsArray, parameter));
return;
}
case OPERATOR_GREATER_EQUAL: {
logger.info("Greater than equal to operator. Is with byte array - " + byteArray);
filters.add(getCellFilter(filterConfig, CompareOp.GREATER_OR_EQUAL, !byteArray ? Bytes.toBytes(
parameter.toString()) : paramAsArray, parameter));
return;
}
case OPERATOR_NOT_EQUAL: {
filters.add(getCellFilter(filterConfig, CompareOp.NOT_EQUAL, !byteArray ? Bytes.toBytes(parameter.toString()) :
paramAsArray, parameter));
return;
}
case OPERATOR_IS_EMPTY:
case OPERATOR_IS_NULL: {
final Filter cellFilter = getCellFilter(filterConfig, CompareOp.EQUAL, Bytes.toBytes(""), parameter);
if (cellFilter instanceof SingleColumnValueFilter) {
((SingleColumnValueFilter) cellFilter).setFilterIfMissing(false);
}
filters.add(cellFilter);
return;
}
case OPERATOR_IS_NOT_EMPTY:
case OPERATOR_IS_NOT_NULL: {
final Filter cellFilter = getCellFilter(filterConfig, CompareOp.NOT_EQUAL, Bytes.toBytes(""), parameter);
if (cellFilter instanceof SingleColumnValueFilter) {
((SingleColumnValueFilter) cellFilter).setFilterIfMissing(true);
}
filters.add(cellFilter);
return;
}
case OPERATOR_STRING_LIKE: {
MatchMode matchMode = getMatchMode(queryParameter);
if (matchMode == null) {
matchMode = MatchMode.EXACT;
}
switch (matchMode) {
case END:
logger.info("String like end operator. Is with byte array - " + byteArray);
filters.add(getCellFilter(filterConfig, CompareOp.EQUAL, new RegexStringComparator(!byteArray ? parameter.
toString() : new String(paramAsArray)), parameter));
break;
case EXACT:
filters.add(getCellFilter(filterConfig, CompareOp.EQUAL, new BinaryComparator(!byteArray ? Bytes.toBytes(parameter.
toString()) : paramAsArray), parameter));
break;
case START:
logger.info("String like start operator. Is with byte array - " + byteArray);
filters.add(getCellFilter(filterConfig, CompareOp.EQUAL, new BinaryPrefixComparator(!byteArray ? Bytes.
toBytes(parameter.toString()) : paramAsArray), parameter));
break;
default:
case ANYWHERE:
filters.add(getCellFilter(filterConfig, CompareOp.EQUAL, new SubstringComparator(parameter.toString()),
parameter));
break;
}
return;
}
case OPERATOR_BETWEEN: {
parameter = getFirstParameter(queryParameter);
Object parameter2 = getSecondParameter(queryParameter);
final boolean byteArray2;
final byte[] paramAsArray2;
if (parameter2 instanceof byte[]) {
paramAsArray2 = (byte[]) parameter2;
byteArray2 = true;
}
else {
paramAsArray2 = null;
byteArray2 = false;
}
filters.add(
getCellFilter(filterConfig, CompareOp.EQUAL,
new RangeComparator(!byteArray ? Bytes.toBytes(parameter.toString()) : paramAsArray,
byteArray2 ? Bytes.toBytes(parameter2.toString()) : paramAsArray2), null));
return;
}
case OPERATOR_IS_IN: {
Collection inCollectin = QueryParameterCastHelper.MULTI_OPERAND_PARAM_HELPER.cast(queryParameter).getValues();
Filter filterList = getInFilter(inCollectin, filterConfig);
filters.add(filterList);
return;
}
case OPERATOR_IS_NOT_IN: {
Collection notInCollectin = QueryParameterCastHelper.MULTI_OPERAND_PARAM_HELPER.cast(queryParameter).getValues();
Filter filterList = getInFilter(notInCollectin, filterConfig);
filters.add(new SkipFilter(filterList));
return;
}
}
return;
}
protected Filter getCellFilter(FilterConfig filterConfig, CompareOp op,
WritableByteArrayComparable comparator, Object originalParam) {
if (originalParam instanceof Filter) {
logger.info("Found Filter from client so using that!");
return (Filter) originalParam;
}
if (filterConfig.isFilterOnRowId()) {
logger.info("Filtering on row id!");
RowFilter rowFilter = new RowFilter(op, comparator);
return rowFilter;
}
else if (filterConfig.isQualifierARangePrefix()) {
logger.info("Filtering on range prefix qualifier!");
QualifierFilter filter = new QualifierFilter(op, comparator);
return filter;
}
else {
logger.info("Filtering on a cell!");
final SingleColumnValueFilter valueFilter;
valueFilter = new SingleColumnValueFilter(filterConfig.getColumnFamily(),
filterConfig.getColumnQualifier(),
op, comparator);
valueFilter.setFilterIfMissing(filterConfig.isFilterOnIfMissing());
valueFilter.setLatestVersionOnly(filterConfig.isFilterOnLatestVersionOnly());
return valueFilter;
}
}
protected Filter getCellFilter(FilterConfig filterConfig, CompareOp op, byte[] value, Object originalParam) {
return getCellFilter(filterConfig, op, new BinaryComparator(value), originalParam);
}
protected Filter getInFilter(Collection inCollectin, FilterConfig config) {
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
for (Object inObj : inCollectin) {
final boolean byteArray;
final byte[] paramAsArray;
if (inObj instanceof byte[]) {
paramAsArray = (byte[]) inObj;
byteArray = true;
}
else {
paramAsArray = null;
byteArray = false;
}
filterList.addFilter(getCellFilter(config, CompareOp.EQUAL, new BinaryComparator(!byteArray ? Bytes.toBytes(inObj.
toString()) : paramAsArray), null));
}
return filterList;
}
protected String getPropertyName(String prefix, QueryParameter param) {
final StringBuilder propertyName = new StringBuilder("");
if (StringUtils.isNotBlank(prefix)) {
propertyName.append(prefix).append('.');
}
if (param instanceof QueryParameterWithPropertyName) {
propertyName.append(((QueryParameterWithPropertyName) param).getPropertyName());
}
return propertyName.toString();
}
protected Object getSecondParameter(QueryParameter queryParamemter) {
if (queryParamemter instanceof BiOperandQueryParameter) {
return QueryParameterCastHelper.BI_OPERAND_PARAM_HELPER.cast(queryParamemter).getSecondValue();
}
else {
return "";
}
}
protected Object getFirstParameter(QueryParameter queryParamemter) {
if (queryParamemter instanceof BiOperandQueryParameter) {
return QueryParameterCastHelper.BI_OPERAND_PARAM_HELPER.cast(queryParamemter).getFirstValue();
}
else {
return "";
}
}
protected MatchMode getMatchMode(QueryParameter queryParamemter) {
return QueryParameterCastHelper.STRING_PARAM_HELPER.cast(queryParamemter).getMatchMode();
}
protected Object getValue(QueryParameter queryParamemter) {
Object value;
if (queryParamemter instanceof QueryParameterWithValue) {
value = ((QueryParameterWithValue) queryParamemter).getValue();
}
else {
value = null;
}
if (value == null) {
value = "";
}
return value;
}
protected OperatorType getOperator(QueryParameter queryParamemter) {
if (QueryParameterCastHelper.BI_OPERAND_PARAM_HELPER.isWithOperator(queryParamemter)) {
QueryParameterWithOperator parameterWithOperator =
QueryParameterCastHelper.BI_OPERAND_PARAM_HELPER.castToOperatorParam(queryParamemter);
return parameterWithOperator.getOperatorType();
}
else {
return null;
}
}
@Override
public Template getSingle(QueryParameter... query) {
return getSingle(Arrays.asList(query));
}
@Override
public List<Template> getList(QueryParameter... query) {
return getList(Arrays.asList(query));
}
@Override
public <OtherTemplate> OtherTemplate getOther(QueryParameter... query) {
return this.<OtherTemplate>getOther(Arrays.asList(query));
}
@Override
public <OtherTemplate> List<OtherTemplate> getOtherList(QueryParameter... query) {
return this.<OtherTemplate>getOtherList(Arrays.asList(query));
}
/*
* WRITE OPERATIONS
*/
@Override
public void save(Template... states) {
verifyAllEntitiesExists(false, states);
put(states, false);
}
@Override
public void update(Template... states) {
verifyAllEntitiesExists(true, states);
put(states, true);
}
protected void throwIfErrors(Collection<Future<String>> probableFutureErrors) throws IllegalStateException {
Collection<String> probableErrors = new ArrayList<String>(probableFutureErrors.size());
for (Future<String> delErr : probableFutureErrors) {
String str;
try {
str = delErr.get();
}
catch (Exception ex) {
logger.warn("Could not wait to complete deletion!", ex);
str = "Could not complete deletion!";
}
if (StringUtils.isNotBlank(str)) {
probableErrors.add(str);
}
}
if (!probableErrors.isEmpty()) {
throw new IllegalStateException(Arrays.toString(probableErrors.toArray(new String[probableErrors.size()])));
}
logger.info("No error messages!");
}
protected void verifyAllEntitiesExists(boolean existenceExpected, Template... states) {
List<Future<Boolean>> gets = new ArrayList<Future<Boolean>>(states.length);
int i = 0;
for (Template t : states) {
try {
final Comparable id = t.getId();
if (logger.isInfoEnabled()) {
logger.info(new StringBuilder("Adding ").append(id).append(" to index ").append(i).toString());
}
final Get get = new Get(infoProvider.getRowIdFromId((IdType) id));
final int index = i;
gets.add(executorService.executeAsynchronously(infoProvider.getMainTableName(),
new Callback<Boolean>() {
@Override
public Boolean call(HTableInterface tableInterface) throws Exception {
final boolean exists = tableInterface.exists(get);
if (logger.isDebugEnabled()) {
logger.debug(new StringBuilder("Id ").append(id.toString()).append(" exists ").append(index).toString());
}
return exists;
}
}));
i++;
}
catch (Exception ex) {
logger.warn("Exception testing row existense...", ex);
}
}
i = 0;
for (Future<Boolean> future : gets) {
Boolean exists = true;
try {
if (logger.isInfoEnabled()) {
logger.info(new StringBuilder("Checking index ").append(i++).toString());
}
exists = future.get();
}
catch (Exception ex) {
logger.warn("Exception testing row existense...", ex);
}
if (!existenceExpected && exists) {
throw new IllegalArgumentException(
"Some of the entities are already saved, so did not procced with any of them");
}
if (existenceExpected && !exists) {
throw new IllegalArgumentException("Some of the entities are not saved, so did not procced with any of them");
}
}
}
protected void put(Template[] states, final boolean merge) throws IllegalStateException {
LinkedHashMap<String, List<Put>> allPuts =
new LinkedHashMap<String, List<Put>>();
for (Template state : states) {
if (!state.isValid()) {
throw new IllegalStateException("Entity not in valid state!");
}
final LinkedHashMap<String, Put> puts;
puts = getConverter().objectToRows(state, executorService, getLockType().equals(LockType.PESSIMISTIC));
for (Map.Entry<String, Put> put : puts.entrySet()) {
final List<Put> putList;
if (allPuts.containsKey(put.getKey())) {
putList = allPuts.get(put.getKey());
}
else {
putList = new ArrayList<Put>();
allPuts.put(put.getKey(), putList);
}
putList.add(put.getValue());
}
}
for (Map.Entry<String, List<Put>> puts : allPuts.entrySet()) {
if (LockType.OPTIMISTIC.equals(getLockType()) && infoProvider.getVersionColumnFamily() != null && infoProvider.
getVersionColumnQualifier() != null) {
putOptimistically(puts, merge, states);
}
else {
putNonOptimistically(puts, merge, states);
}
}
}
protected void putOptimistically(Entry<String, List<Put>> puts, final boolean merge, Template[] states) {
final List<Put> value = puts.getValue();
//Merge first respecting lock type
executorService.execute(puts.getKey(),
new Callback<Void>() {
@Override
public Void call(HTableInterface tableInterface) throws Exception {
if (merge && mergeEnabled && mergeService != null) {
mergeService.merge(tableInterface, value, getLockType());
}
return null;
}
});
Collection<Future<String>> pFutures = new ArrayList<Future<String>>();
final byte[] family = infoProvider.getVersionColumnFamily();
final byte[] qualifier = infoProvider.getVersionColumnQualifier();
for (final Put put : puts.getValue()) {
final List<KeyValue> kVal = put.get(family, qualifier);
final byte[] versionValue;
final byte[] nextVersion;
if (kVal != null && !kVal.isEmpty()) {
versionValue = DiffBasedMergeService.getLatestValue(kVal).getValue();
nextVersion = Bytes.toBytes(Bytes.toLong(versionValue) + 1);
}
else {
versionValue = null;
nextVersion = Bytes.toBytes(1l);
}
put.add(family, qualifier, nextVersion);
pFutures.add(executorService.executeAsynchronously(puts.getKey(), new Callback<String>() {
@Override
public String call(HTableInterface tableInterface) throws Exception {
boolean puted = tableInterface.checkAndPut(put.getRow(), family, qualifier, versionValue, put);
if (!puted) {
return String.format(errorMessageFormat, infoProvider.getIdFromRowId(put.getRow()).toString(), Bytes.
toString(tableInterface.getTableName()));
}
return null;
}
}));
}
throwIfErrors(pFutures);
}
protected void putNonOptimistically(Entry<String, List<Put>> puts, final boolean merge, Template[] states) {
try {
final List<Put> value = puts.getValue();
executorService.execute(puts.getKey(), new Callback<Void>() {
@Override
public Void call(HTableInterface tableInterface) throws Exception {
if (merge && mergeEnabled && mergeService != null) {
mergeService.merge(tableInterface, value, getLockType());
}
final ArrayList<Put> valueCopy = new ArrayList<Put>(value);
tableInterface.put(valueCopy);
return null;
}
});
}
finally {
for (Template state : states) {
lockAttainer.unlockAndEvictFromCache(state);
}
}
}
@Override
public void delete(Template... states) {
verifyAllEntitiesExists(true, states);
final byte[] family = infoProvider.getVersionColumnFamily();
final byte[] qualifier = infoProvider.getVersionColumnQualifier();
if (LockType.OPTIMISTIC.equals(getLockType()) && family != null && qualifier != null) {
deleteOptimistically(states);
}
else {
deleteNonOptimistically(states);
}
}
protected void deleteOptimistically(Template[] states) throws IllegalStateException {
final byte[] family = infoProvider.getVersionColumnFamily();
final byte[] qualifier = infoProvider.getVersionColumnQualifier();
Collection<Future<String>> deletes = new ArrayList<Future<String>>();
for (final Template state : states) {
if (!state.isValid()) {
throw new IllegalStateException("Entity not in valid state!");
}
LinkedHashMap<String, Delete> dels = getConverter().objectToDeleteableRows(state, executorService, false);
final byte[] version = state.getVersion() != null ? Bytes.toBytes(state.getVersion()) : null;
if (logger.isInfoEnabled() && version != null) {
logger.info("Version to check on delete optimistically is " + Bytes.toLong(version));
}
else if (logger.isInfoEnabled()) {
logger.info("Version is null");
}
for (final Map.Entry<String, Delete> del : dels.entrySet()) {
deletes.add(executorService.executeAsynchronously(del.getKey(), new Callback<String>() {
@Override
public String call(HTableInterface tableInterface) throws Exception {
final Delete delVal = del.getValue();
boolean deleted = tableInterface.checkAndDelete(delVal.getRow(), family, qualifier, version, delVal);
if (logger.isInfoEnabled()) {
logger.info("Deleted row? " + deleted);
}
if (!deleted) {
return String.format(errorMessageFormat, infoProvider.getIdFromRowId(delVal.getRow()).toString(), Bytes.
toString(tableInterface.getTableName()));
}
return null;
}
}));
}
}
throwIfErrors(deletes);
}
protected void deleteNonOptimistically(Template[] states) throws IllegalStateException {
LinkedHashMap<String, List<Delete>> allDels =
new LinkedHashMap<String, List<Delete>>();
for (Template state : states) {
if (!state.isValid()) {
throw new IllegalStateException("Entity not in valid state!");
}
LinkedHashMap<String, Delete> dels = getConverter().objectToDeleteableRows(state, executorService, getLockType().
equals(LockType.PESSIMISTIC));
for (Map.Entry<String, Delete> del : dels.entrySet()) {
final List<Delete> putList;
if (allDels.containsKey(del.getKey())) {
putList = allDels.get(del.getKey());
}
else {
putList = new ArrayList<Delete>();
allDels.put(del.getKey(), putList);
}
putList.add(del.getValue());
}
}
for (final Map.Entry<String, List<Delete>> dels : allDels.entrySet()) {
try {
executorService.execute(dels.getKey(),
new Callback<Void>() {
@Override
public Void call(HTableInterface tableInterface) throws Exception {
final List<Delete> value = dels.getValue();
if (logger.isInfoEnabled()) {
logger.info("Attempting to DELETE " + value);
}
final ArrayList<Delete> list = new ArrayList<Delete>(value);
tableInterface.delete(list);
return null;
}
});
}
finally {
for (Template state : states) {
lockAttainer.unlockAndEvictFromCache(state);
}
}
}
}
}