/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.accumulo.index;
import com.facebook.presto.accumulo.Types;
import com.facebook.presto.accumulo.iterators.MaxByteArrayCombiner;
import com.facebook.presto.accumulo.iterators.MinByteArrayCombiner;
import com.facebook.presto.accumulo.metadata.AccumuloTable;
import com.facebook.presto.accumulo.model.AccumuloColumnHandle;
import com.facebook.presto.accumulo.serializers.AccumuloRowSerializer;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.type.Type;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.primitives.UnsignedBytes;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.ColumnUpdate;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.TypedValueCombiner;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.io.Text;
import javax.annotation.concurrent.NotThreadSafe;
import java.io.Closeable;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import static com.facebook.presto.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE;
import static com.facebook.presto.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
import static com.google.common.base.MoreObjects.toStringHelper;
import static java.nio.ByteBuffer.wrap;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
/**
* This utility class assists the Presto connector, and external applications,
* in populating the index table and metrics table for Accumulo-backed Presto tables.
* <p>
* This class is totally not thread safe.
* <p>
* When creating a table, if it contains indexed columns, users will have to create the index table
* and the index metrics table, the names of which can be retrieved using the static functions in
* this class. Additionally, users MUST add iterators to the index metrics table (also available via
* static function), and, while not required, recommended to add the locality groups to the index
* table to improve index lookup times.
* <p>
* Sample usage of an indexer:
* <p>
* <pre>
* <code>
* Indexer indexer = new Indexer(connector, userAuths, table, writerConf);
* for (Mutation m : mutationsToNormalTable) {
* indexer.index(m);
* }
*
* // can flush indexer w/regular BatchWriter
* indexer.flush()
*
* // finished adding new mutations, close the indexer
* indexer.close();
* </code>
* </pre>
*/
@NotThreadSafe
public class Indexer
implements Closeable
{
public static final ByteBuffer METRICS_TABLE_ROW_ID = wrap("___METRICS_TABLE___".getBytes(UTF_8));
public static final ByteBuffer METRICS_TABLE_ROWS_CF = wrap("___rows___".getBytes(UTF_8));
public static final MetricsKey METRICS_TABLE_ROW_COUNT = new MetricsKey(METRICS_TABLE_ROW_ID, METRICS_TABLE_ROWS_CF);
public static final ByteBuffer METRICS_TABLE_FIRST_ROW_CQ = wrap("___first_row___".getBytes(UTF_8));
public static final ByteBuffer METRICS_TABLE_LAST_ROW_CQ = wrap("___last_row___".getBytes(UTF_8));
public static final byte[] CARDINALITY_CQ = "___card___".getBytes(UTF_8);
public static final Text CARDINALITY_CQ_AS_TEXT = new Text(CARDINALITY_CQ);
public static final Text METRICS_TABLE_ROWS_CF_AS_TEXT = new Text(METRICS_TABLE_ROWS_CF.array());
public static final Text METRICS_TABLE_ROWID_AS_TEXT = new Text(METRICS_TABLE_ROW_ID.array());
private static final byte[] EMPTY_BYTES = new byte[0];
private static final byte UNDERSCORE = '_';
private static final TypedValueCombiner.Encoder<Long> ENCODER = new LongCombiner.StringEncoder();
private final AccumuloTable table;
private final BatchWriter indexWriter;
private final BatchWriterConfig writerConfig;
private final Connector connector;
private final Map<MetricsKey, AtomicLong> metrics = new HashMap<>();
private final Multimap<ByteBuffer, ByteBuffer> indexColumns;
private final Map<ByteBuffer, Map<ByteBuffer, Type>> indexColumnTypes;
private final AccumuloRowSerializer serializer;
private final Comparator<byte[]> byteArrayComparator = UnsignedBytes.lexicographicalComparator();
private byte[] firstRow = null;
private byte[] lastRow = null;
public Indexer(
Connector connector,
Authorizations auths,
AccumuloTable table,
BatchWriterConfig writerConfig)
throws TableNotFoundException
{
this.connector = requireNonNull(connector, "connector is null");
this.table = requireNonNull(table, "table is null");
this.writerConfig = requireNonNull(writerConfig, "writerConfig is null");
requireNonNull(auths, "auths is null");
this.serializer = table.getSerializerInstance();
// Create our batch writer
indexWriter = connector.createBatchWriter(table.getIndexTableName(), writerConfig);
ImmutableMultimap.Builder<ByteBuffer, ByteBuffer> indexColumnsBuilder = ImmutableMultimap.builder();
Map<ByteBuffer, Map<ByteBuffer, Type>> indexColumnTypesBuilder = new HashMap<>();
// Initialize metadata
table.getColumns().forEach(columnHandle -> {
if (columnHandle.isIndexed()) {
// Wrap the column family and qualifier for this column and add it to
// collection of indexed columns
ByteBuffer family = wrap(columnHandle.getFamily().get().getBytes(UTF_8));
ByteBuffer qualifier = wrap(columnHandle.getQualifier().get().getBytes(UTF_8));
indexColumnsBuilder.put(family, qualifier);
// Create a mapping for this column's Presto type, again creating a new one for the
// family if necessary
Map<ByteBuffer, Type> types = indexColumnTypesBuilder.get(family);
if (types == null) {
types = new HashMap<>();
indexColumnTypesBuilder.put(family, types);
}
types.put(qualifier, columnHandle.getType());
}
});
indexColumns = indexColumnsBuilder.build();
indexColumnTypes = ImmutableMap.copyOf(indexColumnTypesBuilder);
// If there are no indexed columns, throw an exception
if (indexColumns.isEmpty()) {
throw new PrestoException(NOT_SUPPORTED, "No indexed columns in table metadata. Refusing to index a table with no indexed columns");
}
// Initialize metrics map
// This metrics map is for column cardinality
metrics.put(METRICS_TABLE_ROW_COUNT, new AtomicLong(0));
// Scan the metrics table for existing first row and last row
Pair<byte[], byte[]> minmax = getMinMaxRowIds(connector, table, auths);
firstRow = minmax.getLeft();
lastRow = minmax.getRight();
}
/**
* Index the given mutation, adding mutations to the index and metrics table
* <p>
* Like typical use of a BatchWriter, this method does not flush mutations to the underlying index table.
* For higher throughput the modifications to the metrics table are tracked in memory and added to the metrics table when the indexer is flushed or closed.
*
* @param mutation Mutation to index
*/
public void index(Mutation mutation)
{
// Increment the cardinality for the number of rows in the table
metrics.get(METRICS_TABLE_ROW_COUNT).incrementAndGet();
// Set the first and last row values of the table based on existing row IDs
if (firstRow == null || byteArrayComparator.compare(mutation.getRow(), firstRow) < 0) {
firstRow = mutation.getRow();
}
if (lastRow == null || byteArrayComparator.compare(mutation.getRow(), lastRow) > 0) {
lastRow = mutation.getRow();
}
// For each column update in this mutation
for (ColumnUpdate columnUpdate : mutation.getUpdates()) {
// Get the column qualifiers we want to index for this column family (if any)
ByteBuffer family = wrap(columnUpdate.getColumnFamily());
Collection<ByteBuffer> indexQualifiers = indexColumns.get(family);
// If we have column qualifiers we want to index for this column family
if (indexQualifiers != null) {
// Check if we want to index this particular qualifier
ByteBuffer qualifier = wrap(columnUpdate.getColumnQualifier());
if (indexQualifiers.contains(qualifier)) {
// If so, create a mutation using the following mapping:
// Row ID = column value
// Column Family = columnqualifier_columnfamily
// Column Qualifier = row ID
// Value = empty
ByteBuffer indexFamily = getIndexColumnFamily(columnUpdate.getColumnFamily(), columnUpdate.getColumnQualifier());
Type type = indexColumnTypes.get(family).get(qualifier);
ColumnVisibility visibility = new ColumnVisibility(columnUpdate.getColumnVisibility());
// If this is an array type, then index each individual element in the array
if (Types.isArrayType(type)) {
Type elementType = Types.getElementType(type);
List<?> elements = serializer.decode(type, columnUpdate.getValue());
for (Object element : elements) {
addIndexMutation(wrap(serializer.encode(elementType, element)), indexFamily, visibility, mutation.getRow());
}
}
else {
addIndexMutation(wrap(columnUpdate.getValue()), indexFamily, visibility, mutation.getRow());
}
}
}
}
}
public void index(Iterable<Mutation> mutations)
{
for (Mutation mutation : mutations) {
index(mutation);
}
}
private void addIndexMutation(ByteBuffer row, ByteBuffer family, ColumnVisibility visibility, byte[] qualifier)
{
// Create the mutation and add it to the batch writer
Mutation indexMutation = new Mutation(row.array());
indexMutation.put(family.array(), qualifier, visibility, EMPTY_BYTES);
try {
indexWriter.addMutation(indexMutation);
}
catch (MutationsRejectedException e) {
throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Index mutation rejected by server", e);
}
// Increment the cardinality metrics for this value of index
// metrics is a mapping of row ID to column family
MetricsKey key = new MetricsKey(row, family, visibility);
AtomicLong count = metrics.get(key);
if (count == null) {
count = new AtomicLong(0);
metrics.put(key, count);
}
count.incrementAndGet();
}
/**
* Flushes all Mutations in the index writer. And all metric mutations to the metrics table.
* Note that the metrics table is not updated until this method is explicitly called (or implicitly via close).
*/
public void flush()
{
try {
// Flush index writer
indexWriter.flush();
// Write out metrics mutations
BatchWriter metricsWriter = connector.createBatchWriter(table.getMetricsTableName(), writerConfig);
metricsWriter.addMutations(getMetricsMutations());
metricsWriter.close();
// Re-initialize the metrics
metrics.clear();
metrics.put(METRICS_TABLE_ROW_COUNT, new AtomicLong(0));
}
catch (MutationsRejectedException e) {
throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Index mutation was rejected by server on flush", e);
}
catch (TableNotFoundException e) {
throw new PrestoException(ACCUMULO_TABLE_DNE, "Accumulo table does not exist", e);
}
}
/**
* Flushes all remaining mutations via {@link Indexer#flush} and closes the index writer.
*/
@Override
public void close()
{
try {
flush();
indexWriter.close();
}
catch (MutationsRejectedException e) {
throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation was rejected by server on close", e);
}
}
private Collection<Mutation> getMetricsMutations()
{
ImmutableList.Builder<Mutation> mutationBuilder = ImmutableList.builder();
// Mapping of column value to column to number of row IDs that contain that value
for (Entry<MetricsKey, AtomicLong> entry : metrics.entrySet()) {
// Row ID: Column value
// Family: columnfamily_columnqualifier
// Qualifier: CARDINALITY_CQ
// Visibility: Inherited from indexed Mutation
// Value: Cardinality
Mutation mut = new Mutation(entry.getKey().row.array());
mut.put(entry.getKey().family.array(), CARDINALITY_CQ, entry.getKey().visibility, ENCODER.encode(entry.getValue().get()));
// Add to our list of mutations
mutationBuilder.add(mut);
}
// If the first row and last row are both not null,
// which would really be for a brand new table that has zero rows and no indexed elements...
// Talk about your edge cases!
if (firstRow != null && lastRow != null) {
// Add a some columns to the special metrics table row ID for the first/last row.
// Note that if the values on the server side are greater/lesser,
// the configured iterator will take care of this at scan/compaction time
Mutation firstLastMutation = new Mutation(METRICS_TABLE_ROW_ID.array());
firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_FIRST_ROW_CQ.array(), firstRow);
firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_LAST_ROW_CQ.array(), lastRow);
mutationBuilder.add(firstLastMutation);
}
return mutationBuilder.build();
}
/**
* Gets a collection of iterator settings that should be added to the metric table for the given Accumulo table. Don't forget! Please!
*
* @param table Table for retrieving metrics iterators, see AccumuloClient#getTable
* @return Collection of iterator settings
*/
public static Collection<IteratorSetting> getMetricIterators(AccumuloTable table)
{
String cardQualifier = new String(CARDINALITY_CQ);
String rowsFamily = new String(METRICS_TABLE_ROWS_CF.array());
// Build a string for all columns where the summing combiner should be applied,
// i.e. all indexed columns
StringBuilder cardBuilder = new StringBuilder(rowsFamily + ":" + cardQualifier + ",");
for (String s : getLocalityGroups(table).keySet()) {
cardBuilder.append(s).append(":").append(cardQualifier).append(',');
}
cardBuilder.deleteCharAt(cardBuilder.length() - 1);
// Configuration rows for the Min/Max combiners
String firstRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_FIRST_ROW_CQ.array());
String lastRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_LAST_ROW_CQ.array());
// Summing combiner for cardinality columns
IteratorSetting s1 = new IteratorSetting(1, SummingCombiner.class, ImmutableMap.of("columns", cardBuilder.toString(), "type", "STRING"));
// Min/Max combiner for the first/last rows of the table
IteratorSetting s2 = new IteratorSetting(2, MinByteArrayCombiner.class, ImmutableMap.of("columns", firstRowColumn));
IteratorSetting s3 = new IteratorSetting(3, MaxByteArrayCombiner.class, ImmutableMap.of("columns", lastRowColumn));
return ImmutableList.of(s1, s2, s3);
}
/**
* Gets the column family of the index table based on the given column family and qualifier.
*
* @param columnFamily Presto column family
* @param columnQualifier Presto column qualifier
* @return ByteBuffer of the given index column family
*/
public static ByteBuffer getIndexColumnFamily(byte[] columnFamily, byte[] columnQualifier)
{
return wrap(ArrayUtils.addAll(ArrayUtils.add(columnFamily, UNDERSCORE), columnQualifier));
}
/**
* Gets a set of locality groups that should be added to the index table (not the metrics table).
*
* @param table Table for the locality groups, see AccumuloClient#getTable
* @return Mapping of locality group to column families in the locality group, 1:1 mapping in
* this case
*/
public static Map<String, Set<Text>> getLocalityGroups(AccumuloTable table)
{
Map<String, Set<Text>> groups = new HashMap<>();
// For each indexed column
for (AccumuloColumnHandle columnHandle : table.getColumns().stream().filter(AccumuloColumnHandle::isIndexed).collect(Collectors.toList())) {
// Create a Text version of the index column family
Text indexColumnFamily = new Text(getIndexColumnFamily(columnHandle.getFamily().get().getBytes(UTF_8), columnHandle.getQualifier().get().getBytes(UTF_8)).array());
// Add this to the locality groups,
// it is a 1:1 mapping of locality group to column families
groups.put(indexColumnFamily.toString(), ImmutableSet.of(indexColumnFamily));
}
return groups;
}
/**
* Gets the fully-qualified index table name for the given table.
*
* @param schema Schema name
* @param table Table name
* @return Qualified index table name
*/
public static String getIndexTableName(String schema, String table)
{
return schema.equals("default") ? table + "_idx" : schema + '.' + table + "_idx";
}
/**
* Gets the fully-qualified index table name for the given table.
*
* @param tableName Schema table name
* @return Qualified index table name
*/
public static String getIndexTableName(SchemaTableName tableName)
{
return getIndexTableName(tableName.getSchemaName(), tableName.getTableName());
}
/**
* Gets the fully-qualified index metrics table name for the given table.
*
* @param schema Schema name
* @param table Table name
* @return Qualified index metrics table name
*/
public static String getMetricsTableName(String schema, String table)
{
return schema.equals("default") ? table + "_idx_metrics"
: schema + '.' + table + "_idx_metrics";
}
/**
* Gets the fully-qualified index metrics table name for the given table.
*
* @param tableName Schema table name
* @return Qualified index metrics table name
*/
public static String getMetricsTableName(SchemaTableName tableName)
{
return getMetricsTableName(tableName.getSchemaName(), tableName.getTableName());
}
public static Pair<byte[], byte[]> getMinMaxRowIds(Connector connector, AccumuloTable table, Authorizations auths)
throws TableNotFoundException
{
Scanner scanner = connector.createScanner(table.getMetricsTableName(), auths);
scanner.setRange(new Range(new Text(Indexer.METRICS_TABLE_ROW_ID.array())));
Text family = new Text(Indexer.METRICS_TABLE_ROWS_CF.array());
Text firstRowQualifier = new Text(Indexer.METRICS_TABLE_FIRST_ROW_CQ.array());
Text lastRowQualifier = new Text(Indexer.METRICS_TABLE_LAST_ROW_CQ.array());
scanner.fetchColumn(family, firstRowQualifier);
scanner.fetchColumn(family, lastRowQualifier);
byte[] firstRow = null;
byte[] lastRow = null;
for (Entry<Key, Value> entry : scanner) {
if (entry.getKey().compareColumnQualifier(firstRowQualifier) == 0) {
firstRow = entry.getValue().get();
}
if (entry.getKey().compareColumnQualifier(lastRowQualifier) == 0) {
lastRow = entry.getValue().get();
}
}
scanner.close();
return Pair.of(firstRow, lastRow);
}
/**
* Class containing the key for aggregating the local metrics counter.
*/
private static class MetricsKey
{
private static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility();
public final ByteBuffer row;
public final ByteBuffer family;
public final ColumnVisibility visibility;
public MetricsKey(ByteBuffer row, ByteBuffer family)
{
requireNonNull(row, "row is null");
requireNonNull(family, "family is null");
this.row = row;
this.family = family;
this.visibility = EMPTY_VISIBILITY;
}
public MetricsKey(ByteBuffer row, ByteBuffer family, ColumnVisibility visibility)
{
requireNonNull(row, "row is null");
requireNonNull(family, "family is null");
requireNonNull(visibility, "visibility is null");
this.row = row;
this.family = family;
this.visibility = visibility.getExpression() != null ? visibility : EMPTY_VISIBILITY;
}
@Override
public boolean equals(Object obj)
{
if (this == obj) {
return true;
}
if ((obj == null) || (getClass() != obj.getClass())) {
return false;
}
MetricsKey other = (MetricsKey) obj;
return Objects.equals(this.row, other.row)
&& Objects.equals(this.family, other.family)
&& Objects.equals(this.visibility, other.visibility);
}
@Override
public int hashCode()
{
return Objects.hash(row, family, visibility);
}
@Override
public String toString()
{
return toStringHelper(this)
.add("row", new String(row.array(), UTF_8))
.add("family", new String(row.array(), UTF_8))
.add("visibility", visibility.toString())
.toString();
}
}
}