/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.accumulo.index;
import com.facebook.presto.accumulo.conf.AccumuloConfig;
import com.facebook.presto.accumulo.model.AccumuloColumnConstraint;
import com.facebook.presto.spi.PrestoException;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
import io.airlift.concurrent.BoundedExecutor;
import io.airlift.log.Logger;
import io.airlift.units.Duration;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.io.Text;
import javax.annotation.Nonnull;
import javax.annotation.PreDestroy;
import javax.inject.Inject;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
import static com.facebook.presto.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
import static com.facebook.presto.accumulo.index.Indexer.CARDINALITY_CQ_AS_TEXT;
import static com.facebook.presto.accumulo.index.Indexer.getIndexColumnFamily;
import static com.facebook.presto.accumulo.index.Indexer.getMetricsTableName;
import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR;
import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.collect.Streams.stream;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static java.lang.Long.parseLong;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.Executors.newCachedThreadPool;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
/**
* This class is an indexing utility to cache the cardinality of a column value for every table.
* Each table has its own cache that is independent of every other, and every column also has its
* own Guava cache. Use of this utility can have a significant impact for retrieving the cardinality
* of many columns, preventing unnecessary accesses to the metrics table in Accumulo for a
* cardinality that won't change much.
*/
public class ColumnCardinalityCache
{
private static final Logger LOG = Logger.get(ColumnCardinalityCache.class);
private final Connector connector;
private final ExecutorService coreExecutor;
private final BoundedExecutor executorService;
private final LoadingCache<CacheKey, Long> cache;
@Inject
public ColumnCardinalityCache(Connector connector, AccumuloConfig config)
{
this.connector = requireNonNull(connector, "connector is null");
int size = requireNonNull(config, "config is null").getCardinalityCacheSize();
Duration expireDuration = config.getCardinalityCacheExpiration();
// Create a bounded executor with a pool size at 4x number of processors
this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s"));
this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors());
LOG.debug("Created new cache size %d expiry %s", size, expireDuration);
cache = CacheBuilder.newBuilder()
.maximumSize(size)
.expireAfterWrite(expireDuration.toMillis(), MILLISECONDS)
.build(new CardinalityCacheLoader());
}
@PreDestroy
public void shutdown()
{
coreExecutor.shutdownNow();
}
/**
* Gets the cardinality for each {@link AccumuloColumnConstraint}.
* Given constraints are expected to be indexed! Who knows what would happen if they weren't!
*
* @param schema Schema name
* @param table Table name
* @param auths Scan authorizations
* @param idxConstraintRangePairs Mapping of all ranges for a given constraint
* @param earlyReturnThreshold Smallest acceptable cardinality to return early while other tasks complete
* @param pollingDuration Duration for polling the cardinality completion service
* @return An immutable multimap of cardinality to column constraint, sorted by cardinality from smallest to largest
* @throws TableNotFoundException If the metrics table does not exist
* @throws ExecutionException If another error occurs; I really don't even know anymore.
*/
public Multimap<Long, AccumuloColumnConstraint> getCardinalities(String schema, String table, Authorizations auths, Multimap<AccumuloColumnConstraint, Range> idxConstraintRangePairs, long earlyReturnThreshold, Duration pollingDuration)
throws ExecutionException, TableNotFoundException
{
// Submit tasks to the executor to fetch column cardinality, adding it to the Guava cache if necessary
CompletionService<Pair<Long, AccumuloColumnConstraint>> executor = new ExecutorCompletionService<>(executorService);
idxConstraintRangePairs.asMap().forEach((key, value) -> executor.submit(() -> {
long cardinality = getColumnCardinality(schema, table, auths, key.getFamily(), key.getQualifier(), value);
LOG.debug("Cardinality for column %s is %s", key.getName(), cardinality);
return Pair.of(cardinality, key);
}
));
// Create a multi map sorted by cardinality
ListMultimap<Long, AccumuloColumnConstraint> cardinalityToConstraints = MultimapBuilder.treeKeys().arrayListValues().build();
try {
boolean earlyReturn = false;
int numTasks = idxConstraintRangePairs.asMap().entrySet().size();
do {
// Sleep for the polling duration to allow concurrent tasks to run for this time
Thread.sleep(pollingDuration.toMillis());
// Poll each task, retrieving the result if it is done
for (int i = 0; i < numTasks; ++i) {
Future<Pair<Long, AccumuloColumnConstraint>> futureCardinality = executor.poll();
if (futureCardinality != null && futureCardinality.isDone()) {
Pair<Long, AccumuloColumnConstraint> columnCardinality = futureCardinality.get();
cardinalityToConstraints.put(columnCardinality.getLeft(), columnCardinality.getRight());
}
}
// If the smallest cardinality is present and below the threshold, set the earlyReturn flag
Optional<Entry<Long, AccumuloColumnConstraint>> smallestCardinality = cardinalityToConstraints.entries().stream().findFirst();
if (smallestCardinality.isPresent()) {
if (smallestCardinality.get().getKey() <= earlyReturnThreshold) {
LOG.info("Cardinality %s, is below threshold. Returning early while other tasks finish", smallestCardinality);
earlyReturn = true;
}
}
}
while (!earlyReturn && cardinalityToConstraints.entries().size() < numTasks);
}
catch (ExecutionException | InterruptedException e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Exception when getting cardinality", e);
}
// Create a copy of the cardinalities
return ImmutableMultimap.copyOf(cardinalityToConstraints);
}
/**
* Gets the column cardinality for all of the given range values. May reach out to the
* metrics table in Accumulo to retrieve new cache elements.
*
* @param schema Table schema
* @param table Table name
* @param auths Scan authorizations
* @param family Accumulo column family
* @param qualifier Accumulo column qualifier
* @param colValues All range values to summarize for the cardinality
* @return The cardinality of the column
*/
public long getColumnCardinality(String schema, String table, Authorizations auths, String family, String qualifier, Collection<Range> colValues)
throws ExecutionException
{
LOG.debug("Getting cardinality for %s:%s", family, qualifier);
// Collect all exact Accumulo Ranges, i.e. single value entries vs. a full scan
Collection<CacheKey> exactRanges = colValues.stream()
.filter(ColumnCardinalityCache::isExact)
.map(range -> new CacheKey(schema, table, family, qualifier, range, auths))
.collect(Collectors.toList());
LOG.debug("Column values contain %s exact ranges of %s", exactRanges.size(), colValues.size());
// Sum the cardinalities for the exact-value Ranges
// This is where the reach-out to Accumulo occurs for all Ranges that have not
// previously been fetched
long sum = cache.getAll(exactRanges).values().stream().mapToLong(Long::longValue).sum();
// If these collection sizes are not equal,
// then there is at least one non-exact range
if (exactRanges.size() != colValues.size()) {
// for each range in the column value
for (Range range : colValues) {
// if this range is not exact
if (!isExact(range)) {
// Then get the value for this range using the single-value cache lookup
sum += cache.get(new CacheKey(schema, table, family, qualifier, range, auths));
}
}
}
return sum;
}
private static boolean isExact(Range range)
{
return !range.isInfiniteStartKey() && !range.isInfiniteStopKey() &&
range.getStartKey().followingKey(PartialKey.ROW).equals(range.getEndKey());
}
/**
* Complex key for the CacheLoader
*/
private static class CacheKey
{
private final String schema;
private final String table;
private final String family;
private final String qualifier;
private final Range range;
private final Authorizations auths;
public CacheKey(
String schema,
String table,
String family,
String qualifier,
Range range,
Authorizations auths)
{
this.schema = requireNonNull(schema, "schema is null");
this.table = requireNonNull(table, "table is null");
this.family = requireNonNull(family, "family is null");
this.qualifier = requireNonNull(qualifier, "qualifier is null");
this.range = requireNonNull(range, "range is null");
this.auths = requireNonNull(auths, "auths is null");
}
public String getSchema()
{
return schema;
}
public String getTable()
{
return table;
}
public String getFamily()
{
return family;
}
public String getQualifier()
{
return qualifier;
}
public Range getRange()
{
return range;
}
public Authorizations getAuths()
{
return auths;
}
@Override
public int hashCode()
{
return Objects.hash(schema, table, family, qualifier, range);
}
@Override
public boolean equals(Object obj)
{
if (this == obj) {
return true;
}
if ((obj == null) || (getClass() != obj.getClass())) {
return false;
}
CacheKey other = (CacheKey) obj;
return Objects.equals(this.schema, other.schema)
&& Objects.equals(this.table, other.table)
&& Objects.equals(this.family, other.family)
&& Objects.equals(this.qualifier, other.qualifier)
&& Objects.equals(this.range, other.range);
}
@Override
public String toString()
{
return toStringHelper(this)
.add("schema", schema)
.add("table", table)
.add("family", family)
.add("qualifier", qualifier)
.add("range", range).toString();
}
}
/**
* Internal class for loading the cardinality from Accumulo
*/
private class CardinalityCacheLoader
extends CacheLoader<CacheKey, Long>
{
/**
* Loads the cardinality for the given Range. Uses a BatchScanner and sums the cardinality for all values that encapsulate the Range.
*
* @param key Range to get the cardinality for
* @return The cardinality of the column, which would be zero if the value does not exist
*/
@Override
public Long load(@Nonnull CacheKey key)
throws Exception
{
LOG.debug("Loading a non-exact range from Accumulo: %s", key);
// Get metrics table name and the column family for the scanner
String metricsTable = getMetricsTableName(key.getSchema(), key.getTable());
Text columnFamily = new Text(getIndexColumnFamily(key.getFamily().getBytes(UTF_8), key.getQualifier().getBytes(UTF_8)).array());
// Create scanner for querying the range
Scanner scanner = connector.createScanner(metricsTable, key.getAuths());
scanner.setRange(key.getRange());
scanner.fetchColumn(columnFamily, CARDINALITY_CQ_AS_TEXT);
try {
return stream(scanner)
.map(Entry::getValue)
.map(Value::toString)
.mapToLong(Long::parseLong)
.sum();
}
finally {
scanner.close();
}
}
@Override
public Map<CacheKey, Long> loadAll(@Nonnull Iterable<? extends CacheKey> keys)
throws Exception
{
int size = Iterables.size(keys);
if (size == 0) {
return ImmutableMap.of();
}
LOG.debug("Loading %s exact ranges from Accumulo", size);
// In order to simplify the implementation, we are making a (safe) assumption
// that the CacheKeys will all contain the same combination of schema/table/family/qualifier
// This is asserted with the below implementation error just to make sure
CacheKey anyKey = stream(keys).findAny().get();
if (stream(keys).anyMatch(k -> !k.getSchema().equals(anyKey.getSchema()) || !k.getTable().equals(anyKey.getTable()) || !k.getFamily().equals(anyKey.getFamily()) || !k.getQualifier().equals(anyKey.getQualifier()))) {
throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "loadAll called with a non-homogeneous collection of cache keys");
}
Map<Range, CacheKey> rangeToKey = stream(keys).collect(Collectors.toMap(CacheKey::getRange, Function.identity()));
LOG.debug("rangeToKey size is %s", rangeToKey.size());
// Get metrics table name and the column family for the scanner
String metricsTable = getMetricsTableName(anyKey.getSchema(), anyKey.getTable());
Text columnFamily = new Text(getIndexColumnFamily(anyKey.getFamily().getBytes(UTF_8), anyKey.getQualifier().getBytes(UTF_8)).array());
BatchScanner scanner = connector.createBatchScanner(metricsTable, anyKey.getAuths(), 10);
try {
scanner.setRanges(stream(keys).map(CacheKey::getRange).collect(Collectors.toList()));
scanner.fetchColumn(columnFamily, CARDINALITY_CQ_AS_TEXT);
// Create a new map to hold our cardinalities for each range, returning a default of
// Zero for each non-existent Key
Map<CacheKey, Long> rangeValues = new HashMap<>();
stream(keys).forEach(key -> rangeValues.put(key, 0L));
for (Entry<Key, Value> entry : scanner) {
rangeValues.put(rangeToKey.get(Range.exact(entry.getKey().getRow())), parseLong(entry.getValue().toString()));
}
return rangeValues;
}
finally {
if (scanner != null) {
scanner.close();
}
}
}
}
}