package com.linkedin.thirdeye.client;
import com.linkedin.thirdeye.client.pinot.PinotThirdEyeClient;
import com.linkedin.thirdeye.dashboard.configs.CollectionConfig;
import com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO;
import com.linkedin.thirdeye.util.ThirdEyeUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.base.MoreObjects;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.Multimap;
import com.linkedin.thirdeye.api.TimeGranularity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Request object containing all information for a {@link ThirdEyeClient} to retrieve data. Request
* objects can be constructed via {@link ThirdEyeRequestBuilder}.
*/
public class ThirdEyeRequest {
private final List<MetricFunction> metricFunctions;
private final DateTime startTime;
private final DateTime endTime;
private final Multimap<String, String> filterSet;
// TODO - what kind of advanced expressions do we want here? This could potentially force code to
// depend on a specific client implementation
private final String filterClause;
private final List<String> groupByDimensions;
private final TimeGranularity groupByTimeGranularity;
private final List<String> metricNames;
private final String client;
private final String requestReference;
private ThirdEyeRequest(String requestReference, ThirdEyeRequestBuilder builder) {
this.requestReference = requestReference;
this.metricFunctions = builder.metricFunctions;
this.startTime = builder.startTime;
this.endTime = builder.endTime;
this.filterSet = builder.filterSet;
this.filterClause = builder.filterClause;
this.groupByDimensions = builder.groupBy;
this.groupByTimeGranularity = builder.groupByTimeGranularity;
this.client = builder.client;
metricNames = new ArrayList<>();
for (MetricFunction metric : metricFunctions) {
metricNames.add(metric.toString());
}
}
public static ThirdEyeRequestBuilder newBuilder() {
return new ThirdEyeRequestBuilder();
}
public String getRequestReference() {
return requestReference;
}
public List<MetricFunction> getMetricFunctions() {
return metricFunctions;
}
public List<String> getMetricNames() {
return metricNames;
}
@JsonIgnore
public TimeGranularity getGroupByTimeGranularity() {
return groupByTimeGranularity;
}
public DateTime getStartTimeInclusive() {
return startTime;
}
public DateTime getEndTimeExclusive() {
return endTime;
}
public Multimap<String, String> getFilterSet() {
return filterSet;
}
public String getFilterClause() {
// TODO check if this is being used?
return filterClause;
}
public List<String> getGroupBy() {
return groupByDimensions;
}
public String getClient() {
return client;
}
@Override
public int hashCode() {
// TODO do we intentionally omit request reference here?
return Objects.hash(metricFunctions, startTime, endTime, filterSet, filterClause,
groupByDimensions, groupByTimeGranularity);
};
@Override
public boolean equals(Object o) {
if (!(o instanceof ThirdEyeRequest)) {
return false;
}
ThirdEyeRequest other = (ThirdEyeRequest) o;
// TODO do we intentionally omit request reference here?
return Objects.equals(getMetricFunctions(), other.getMetricFunctions())
&& Objects.equals(getStartTimeInclusive(), other.getStartTimeInclusive())
&& Objects.equals(getEndTimeExclusive(), other.getEndTimeExclusive())
&& Objects.equals(getFilterSet(), other.getFilterSet())
&& Objects.equals(getFilterClause(), other.getFilterClause())
&& Objects.equals(getGroupBy(), other.getGroupBy())
&& Objects.equals(getGroupByTimeGranularity(), other.getGroupByTimeGranularity());
};
@Override
public String toString() {
return MoreObjects.toStringHelper(this).add("requestReference", requestReference)
.add("metricFunctions", metricFunctions)
.add("startTime", startTime).add("endTime", endTime).add("filterSet", filterSet)
.add("filterClause", filterClause).add("groupBy", groupByDimensions)
.add("groupByTimeGranularity", groupByTimeGranularity).toString();
}
public static class ThirdEyeRequestBuilder {
private static final Logger LOG = LoggerFactory.getLogger(ThirdEyeRequestBuilder.class);
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
private List<MetricFunction> metricFunctions;
private DateTime startTime;
private DateTime endTime;
private final Multimap<String, String> filterSet;
private String filterClause;
private final List<String> groupBy;
private TimeGranularity groupByTimeGranularity;
private String client;
public ThirdEyeRequestBuilder() {
this.filterSet = LinkedListMultimap.create();
this.groupBy = new ArrayList<String>();
metricFunctions = new ArrayList<>();
this.client = PinotThirdEyeClient.CLIENT_NAME;
}
public ThirdEyeRequestBuilder(ThirdEyeRequest request) {
this.metricFunctions = request.getMetricFunctions();
this.startTime = request.getStartTimeInclusive();
this.endTime = request.getEndTimeExclusive();
this.filterSet = LinkedListMultimap.create(request.getFilterSet());
this.filterClause = request.getFilterClause();
this.groupBy = new ArrayList<String>(request.getGroupBy());
this.groupByTimeGranularity = request.getGroupByTimeGranularity();
this.client = request.getClient();
}
public ThirdEyeRequestBuilder setDatasets(List<String> datasets) {
return this;
}
public ThirdEyeRequestBuilder addMetricFunction(MetricFunction metricFunction) {
metricFunctions.add(metricFunction);
return this;
}
public ThirdEyeRequestBuilder setStartTimeInclusive(long startTimeMillis) {
this.startTime = new DateTime(startTimeMillis, DateTimeZone.UTC);
return this;
}
public ThirdEyeRequestBuilder setStartTimeInclusive(DateTime startTime) {
this.startTime = startTime;
return this;
}
public ThirdEyeRequestBuilder setEndTimeExclusive(long endTimeMillis) {
this.endTime = new DateTime(endTimeMillis, DateTimeZone.UTC);
return this;
}
public ThirdEyeRequestBuilder setEndTimeExclusive(DateTime endTime) {
this.endTime = endTime;
return this;
}
public ThirdEyeRequestBuilder addFilterValue(String column, String... values) {
for (String value : values) {
this.filterSet.put(column, value);
}
return this;
}
public ThirdEyeRequestBuilder setFilterClause(String filterClause) {
this.filterClause = filterClause;
return this;
}
public ThirdEyeRequestBuilder setFilterSet(Multimap<String, String> filterSet) {
if (filterSet != null) {
this.filterSet.clear();
this.filterSet.putAll(filterSet);
}
return this;
}
/** Removes any existing groupings and adds the provided names. */
public ThirdEyeRequestBuilder setGroupBy(Collection<String> names) {
this.groupBy.clear();
addGroupBy(names);
return this;
}
/** See {@link #setGroupBy(Collection)} */
public ThirdEyeRequestBuilder setGroupBy(String... names) {
return setGroupBy(Arrays.asList(names));
}
/** Adds the provided names to the existing groupings. */
public ThirdEyeRequestBuilder addGroupBy(Collection<String> names) {
if (names != null) {
for (String name : names) {
if (name != null) {
this.groupBy.add(name);
}
}
}
return this;
}
/** See {@link ThirdEyeRequestBuilder#addGroupBy(Collection)} */
public ThirdEyeRequestBuilder addGroupBy(String... names) {
return addGroupBy(Arrays.asList(names));
}
public ThirdEyeRequestBuilder setGroupByTimeGranularity(TimeGranularity timeGranularity) {
groupByTimeGranularity = timeGranularity;
return this;
}
public ThirdEyeRequestBuilder setMetricFunctions(List<MetricFunction> metricFunctions) {
this.metricFunctions = metricFunctions;
return this;
}
public void setClient(String client) {
this.client = client;
}
public ThirdEyeRequest build(String requestReference) {
String dataset = null;
// Since we don't have dataset anymore, we are using the first metric function, to derive the dataset name
// and then using that dataset to figure out if non additive
try {
if (CollectionUtils.isNotEmpty(metricFunctions)) {
dataset = ThirdEyeUtils.getDatasetFromMetricFunction(metricFunctions.get(0));
DatasetConfigDTO datasetConfig = ThirdEyeUtils.getDatasetConfigFromName(dataset);
if (!datasetConfig.isAdditive()) {
List<String> collectionDimensionNames = datasetConfig.getDimensions();
decorateFilterSetForPrecomputedDataset(filterSet, groupBy, collectionDimensionNames,
datasetConfig.getDimensionsHaveNoPreAggregation(), datasetConfig.getPreAggregatedKeyword());
}
}
} catch (Exception e) {
LOG.debug("Collection config for collection {} does not exist", dataset);
}
return new ThirdEyeRequest(requestReference, this);
}
/**
* Definition of Pre-Computed Data: the data that has been pre-calculated or pre-aggregated, and does not require
* further aggregation (i.e., aggregation function of Pinot should do no-op). For such data, we assume that there
* exists a dimension value named "all", which is user-definable keyword in collection configuration, that stores
* the pre-aggregated value.
*
* By default, when a query does not specify any value on a certain dimension, Pinot aggregates all values at that
* dimension, which is an undesirable behavior for pre-computed data. Therefore, this method modifies the request's
* dimension filters such that the filter could pick out the "all" value for that dimension.
*
* Example: Suppose that we have a dataset with 3 dimensions: country, pageName, and osName, and the pre-aggregated
* keyword is 'all'. Further assume that the original request's filter = {'country'='US, IN'} and GroupBy dimension =
* pageName, then the decorated request has the new filter = {'country'='US, IN', 'osName' = 'all'}.
*
* @param filterSet the original filterSet. <dt><b>Postconditions:</b><dd> filterSet is decorated with additional
* filters for filtering out the pre-aggregated value on the unspecified dimensions.
*/
public static void decorateFilterSetForPrecomputedDataset(Multimap<String, String> filterSet,
List<String> groupByDimensions, List<String> allDimensions, List<String> dimensionsHaveNoPreAggregation,
String preAggregatedKeyword) {
Set<String> preComputedDimensionNames = new HashSet<>(allDimensions);
if (dimensionsHaveNoPreAggregation.size() != 0) {
preComputedDimensionNames.removeAll(dimensionsHaveNoPreAggregation);
}
Set<String> filterDimensions = filterSet.asMap().keySet();
if (filterDimensions.size() != 0) {
preComputedDimensionNames.removeAll(filterDimensions);
}
if (groupByDimensions.size() != 0) {
preComputedDimensionNames.removeAll(groupByDimensions);
}
if (preComputedDimensionNames.size() != 0) {
for (String preComputedDimensionName : preComputedDimensionNames) {
filterSet.put(preComputedDimensionName, preAggregatedKeyword);
}
}
}
}
}