/**
* CloudGraph Community Edition (CE) License
*
* This is a community release of CloudGraph, a dual-license suite of
* Service Data Object (SDO) 2.1 services designed for relational and
* big-table style "cloud" databases, such as HBase and others.
* This particular copy of the software is released under the
* version 2 of the GNU General Public License. CloudGraph was developed by
* TerraMeta Software, Inc.
*
* Copyright (c) 2013, TerraMeta Software, Inc. All rights reserved.
*
* General License information can be found below.
*
* This distribution may include materials developed by third
* parties. For license and attribution notices for these
* materials, please refer to the documentation that accompanies
* this distribution (see the "Licenses for Third-Party Components"
* appendix) or view the online documentation at
* <http://cloudgraph.org/licenses/>.
*/
package org.cloudgraph.hbase.service;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.util.Bytes;
import org.cloudgraph.hbase.io.TableWriter;
import org.cloudgraph.hbase.mutation.GraphMutationCollector;
import org.cloudgraph.store.service.DuplicateRowException;
import org.cloudgraph.store.service.GraphServiceException;
import org.plasma.sdo.access.DataAccessException;
import org.plasma.sdo.access.DataGraphDispatcher;
import org.plasma.sdo.core.SnapshotMap;
import commonj.sdo.DataGraph;
/**
* Propagates changes to a {@link commonj.sdo.DataGraph data graph} including
* any number of creates (inserts), modifications (updates) and deletes
* across one or more HBase table rows.
* <p>
* For new (created) data graphs, a row key {org.cloudgraph.hbase.key.HBaseRowKeyFactory factory}
* is used to create a new composite HBase row key. The row key generation is
* driven by a configured CloudGraph row key {@link org.cloudgraph.config.RowKeyModel
* model} for a specific HTable {@link org.cloudgraph.config.Table configuration}.
* A minimal set of {@link org.cloudgraph.state.GraphState state} information is
* persisted with each new data graph.
* </p>
* <p>
* For data graphs with any other combination of changes, e.g.
* data object modifications, deletes, etc... an existing HBase
* row key is fetched using an HBase <a href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Get.html" target="#">Get</a>
* operation.
* </p>
* @see org.cloudgraph.hbase.io.DistributedWriter
* @see org.cloudgraph.store.key.GraphRowKeyFactory
* @see org.cloudgraph.store.key.GraphColumnKeyFactory
* @see org.cloudgraph.state.GraphState
*
* @author Scott Cinnamond
* @since 0.5
*/
public class GraphDispatcher extends GraphMutationCollector
implements DataGraphDispatcher
{
private static Log log = LogFactory.getLog(GraphDispatcher.class);
public GraphDispatcher(ServiceContext context, SnapshotMap snapshotMap,
String username) {
super(context, snapshotMap, username);
}
public void close()
{
this.context.close();
}
/**
* Propagates changes to the given <a href="http://docs.plasma-sdo.org/api/org/plasma/sdo/PlasmaDataGraph.html" target="#">data graph</a> including
* any number of creates (inserts), modifications (updates) and deletes
* to a single or multiple HBase tables and table rows.
* @return a map of internally managed concurrency property values and data
* store generated keys.
* @throws DuplicateRowException if for a new data graph, the generated row key
* already exists in the HBase table configured .
*/
public SnapshotMap commit(DataGraph dataGraph) {
if (username == null || username.length() == 0)
throw new IllegalArgumentException("expected username param not, '" + String.valueOf(username) + "'");
else
if (log.isDebugEnabled()) {
log.debug("current user is '" + username + "'");
}
try {
//FIXME: if an exception happens here we don't have table writers to close
// as required by the 1.0.0 HBase client API. Will cause resource bleed
Map<TableWriter, List<Row>> mutations = collectChanges(dataGraph);
TableWriter[] tableWriters = new TableWriter[mutations.keySet().size()];
mutations.keySet().toArray(tableWriters);
try {
this.writeChanges(tableWriters, mutations, this.username);
}
finally {
for (TableWriter tableWriter : tableWriters)
tableWriter.close();
}
return snapshotMap;
}
catch(IOException | IllegalAccessException e) {
throw new DataAccessException(e);
}
}
/**
* Propagates changes to the given array of <a href="http://docs.plasma-sdo.org/api/org/plasma/sdo/PlasmaDataGraph.html" target="#">data graphs</a> including
* any number of creates (inserts), modifications (updates) and deletes
* to a single or multiple HBase tables and table rows. The given graphs may be heterogeneous, with
* different root data objects any 'shape' or depth.
* @return a map of internally managed concurrency property values and data
* store generated keys.
* @throws DuplicateRowException if for a new data graph, the generated row key
* already exists in the HBase table configured .
*/
public SnapshotMap commit(DataGraph[] dataGraphs) {
if (username == null || username.length() == 0)
throw new IllegalArgumentException("expected username param not, '" + String.valueOf(username) + "'");
else
if (log.isDebugEnabled()) {
log.debug("current user is '" + username + "'");
}
try {
// Note: multiple data graphs may share one or more data objects and here we force
// a write for each graph so the state will be saved and re-read from data store
// such that any common data object(s) state will be current.
for (DataGraph dataGraph : dataGraphs) {
Map<TableWriter, List<Row>> mutations = collectChanges(dataGraph);
TableWriter[] tableWriters = new TableWriter[mutations.keySet().size()];
mutations.keySet().toArray(tableWriters);
try {
this.writeChanges(tableWriters, mutations, this.username);
}
finally {
for (TableWriter tableWriter : tableWriters)
tableWriter.close();
}
}
return snapshotMap;
}
catch(IOException | IllegalAccessException e) {
throw new DataAccessException(e);
}
}
private void writeChanges(TableWriter[] tableWriters, Map<TableWriter, List<Row>> mutations, String jobName) throws IOException
{
for (TableWriter tableWriter : tableWriters) {
List<Row> tableMutations = mutations.get(tableWriter);
if (log.isDebugEnabled())
log.debug("commiting "+tableMutations.size()+" mutations to table: " + tableWriter.getTableConfig().getName());
if (log.isDebugEnabled()) {
for (Row row : tableMutations) {
log.debug("commiting "+row.getClass().getSimpleName()+" mutation to table: " + tableWriter.getTableConfig().getName());
debugRowValues(row);
}
}
Object[] results = new Object[tableMutations.size()];
try {
tableWriter.getTable().batch(tableMutations, results);
} catch (InterruptedException e) {
throw new GraphServiceException(e);
}
for (int i = 0; i < results.length; i++) {
if (results[i] == null) {
log.error("batch action (" + i + ") for job '"+jobName+"' failed with null result");
}
else {
if (log.isDebugEnabled())
log.debug("batch action (" + i + ") for job '"+jobName+"' succeeded with "+
String.valueOf(results[i]) + " result");
}
}
//tableWriter.getTable().flushCommits();
//FIXME: find what happened to flush
}
}
private void debugRowValues(Row row)
{
if (row instanceof Mutation) {
Mutation mutation = (Mutation)row;
NavigableMap<byte[], List<Cell>> map = mutation.getFamilyCellMap();
StringBuilder buf = new StringBuilder();
Iterator<byte[]> iter = map.keySet().iterator();
buf.append("[");
int i = 0;
while (iter.hasNext()) {
if (i > 0)
buf.append(", ");
byte[] family = iter.next();
List<Cell> list = map.get(family);
for (Cell cell : list) {
buf.append(Bytes.toString(family));
buf.append(":");
byte[] qual = CellUtil.cloneQualifier(cell);
buf.append(Bytes.toString(qual));
buf.append("=");
byte[] value = CellUtil.cloneValue(cell);
buf.append(Bytes.toString(value));
}
}
buf.append("]");
log.debug("values: "+buf.toString());
}
}
}