/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.operator.aggregation;
import com.facebook.presto.operator.aggregation.state.HyperLogLogState;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.function.AggregationFunction;
import com.facebook.presto.spi.function.AggregationState;
import com.facebook.presto.spi.function.CombineFunction;
import com.facebook.presto.spi.function.InputFunction;
import com.facebook.presto.spi.function.LiteralParameters;
import com.facebook.presto.spi.function.OutputFunction;
import com.facebook.presto.spi.function.SqlType;
import com.facebook.presto.spi.type.StandardTypes;
import com.google.common.annotations.VisibleForTesting;
import io.airlift.slice.Slice;
import io.airlift.stats.cardinality.HyperLogLog;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
import static com.facebook.presto.spi.type.BigintType.BIGINT;
import static com.facebook.presto.util.Failures.checkCondition;
@AggregationFunction("approx_distinct")
public final class ApproximateCountDistinctAggregations
{
private static final double DEFAULT_STANDARD_ERROR = 0.023;
private static final double LOWEST_MAX_STANDARD_ERROR = 0.01150;
private static final double HIGHEST_MAX_STANDARD_ERROR = 0.26000;
private ApproximateCountDistinctAggregations() {}
@InputFunction
public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.BIGINT) long value)
{
input(state, value, DEFAULT_STANDARD_ERROR);
}
@InputFunction
public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.DOUBLE) double maxStandardError)
{
HyperLogLog hll = getOrCreateHyperLogLog(state, maxStandardError);
state.addMemoryUsage(-hll.estimatedInMemorySize());
hll.add(value);
state.addMemoryUsage(hll.estimatedInMemorySize());
}
@InputFunction
public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.DOUBLE) double value)
{
input(state, value, DEFAULT_STANDARD_ERROR);
}
@InputFunction
public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.DOUBLE) double value, @SqlType(StandardTypes.DOUBLE) double maxStandardError)
{
input(state, Double.doubleToLongBits(value), maxStandardError);
}
@InputFunction
@LiteralParameters("x")
public static void input(@AggregationState HyperLogLogState state, @SqlType("varchar(x)") Slice value)
{
input(state, value, DEFAULT_STANDARD_ERROR);
}
@InputFunction
@LiteralParameters("x")
public static void input(@AggregationState HyperLogLogState state, @SqlType("varchar(x)") Slice value, @SqlType(StandardTypes.DOUBLE) double maxStandardError)
{
inputBinary(state, value, maxStandardError);
}
@InputFunction
public static void inputBinary(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.VARBINARY) Slice value)
{
inputBinary(state, value, DEFAULT_STANDARD_ERROR);
}
@InputFunction
public static void inputBinary(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.VARBINARY) Slice value, @SqlType(StandardTypes.DOUBLE) double maxStandardError)
{
HyperLogLog hll = getOrCreateHyperLogLog(state, maxStandardError);
state.addMemoryUsage(-hll.estimatedInMemorySize());
hll.add(value);
state.addMemoryUsage(hll.estimatedInMemorySize());
}
private static HyperLogLog getOrCreateHyperLogLog(HyperLogLogState state, double maxStandardError)
{
HyperLogLog hll = state.getHyperLogLog();
if (hll == null) {
hll = HyperLogLog.newInstance(standardErrorToBuckets(maxStandardError));
state.setHyperLogLog(hll);
state.addMemoryUsage(hll.estimatedInMemorySize());
}
return hll;
}
@VisibleForTesting
static int standardErrorToBuckets(double maxStandardError)
{
checkCondition(maxStandardError >= LOWEST_MAX_STANDARD_ERROR && maxStandardError <= HIGHEST_MAX_STANDARD_ERROR,
INVALID_FUNCTION_ARGUMENT,
"Max standard error must be in [%s, %s]: %s", LOWEST_MAX_STANDARD_ERROR, HIGHEST_MAX_STANDARD_ERROR, maxStandardError);
return log2Ceiling((int) Math.ceil(1.0816 / (maxStandardError * maxStandardError)));
}
private static int log2Ceiling(int value)
{
return Integer.highestOneBit(value - 1) << 1;
}
@CombineFunction
public static void combineState(@AggregationState HyperLogLogState state, @AggregationState HyperLogLogState otherState)
{
HyperLogLog input = otherState.getHyperLogLog();
HyperLogLog previous = state.getHyperLogLog();
if (previous == null) {
state.setHyperLogLog(input);
state.addMemoryUsage(input.estimatedInMemorySize());
}
else {
state.addMemoryUsage(-previous.estimatedInMemorySize());
previous.mergeWith(input);
state.addMemoryUsage(previous.estimatedInMemorySize());
}
}
@OutputFunction(StandardTypes.BIGINT)
public static void evaluateFinal(@AggregationState HyperLogLogState state, BlockBuilder out)
{
HyperLogLog hyperLogLog = state.getHyperLogLog();
if (hyperLogLog == null) {
BIGINT.writeLong(out, 0);
}
else {
BIGINT.writeLong(out, hyperLogLog.cardinality());
}
}
}