package com.linkedin.thirdeye.datalayer.pojo;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.linkedin.thirdeye.api.TimeGranularity;
import com.linkedin.thirdeye.api.TimeSpec;
import com.linkedin.thirdeye.client.pinot.PinotThirdEyeClient;
import com.linkedin.thirdeye.completeness.checker.DataCompletenessConstants.DataCompletenessAlgorithmName;
import com.linkedin.thirdeye.completeness.checker.Wo4WAvgDataCompletenessAlgorithm;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
@JsonIgnoreProperties(ignoreUnknown=true)
public class DatasetConfigBean extends AbstractBean {
public static String DEFAULT_PREAGGREGATED_DIMENSION_VALUE = "all";
public static String DATASET_OFFLINE_PREFIX = "_OFFLINE";
public static TimeGranularity DEFAULT_HOURLY_EXPECTED_DELAY = new TimeGranularity(8, TimeUnit.HOURS);
public static TimeGranularity DEFAULT_DAILY_EXPECTED_DELAY = new TimeGranularity(36, TimeUnit.HOURS);
private String dataset;
private List<String> dimensions;
private String timeColumn;
private TimeUnit timeUnit;
private Integer timeDuration;
private String timeFormat = TimeSpec.SINCE_EPOCH_FORMAT;
private String timezone = TimeSpec.DEFAULT_TIMEZONE;
/** Introduce this as a dataset property because count* queries will have no metric information **/
private String client = PinotThirdEyeClient.CLIENT_NAME;
private boolean metricAsDimension = false;
private String metricNamesColumn;
private String metricValuesColumn;
/** Autodiscover metrics in case of metricAsDimension */
private boolean autoDiscoverMetrics = false;
private boolean active = true;
/** Configuration for non-additive dataset **/
// By default, every dataset is additive and this section of configurations should be ignored.
private boolean additive = true;
// We assume that non-additive dataset has a default TOP dimension value, which is specified via preAggregatedKeyword,
// for each dimension. When ThirdEye constructs query string to backend database, it automatically appends the keyword
// for ALL dimensions except the dimension that has been specified by filter the one specified in
// dimensionsHaveNoPreAggregation (because some dimension may not have such pre-aggregated dimension value).
// For example, assume that we have three dimensions: D1, D2, D3. The preAggregatedKeyword is "all" and D2 does not
// has any pre-aggregated dimension value, i.e., dimensionsHaveNoPreAggregation=[D2]. The filter given by user is
// {D3=V3}. Then the query should append the WHERE condition {D1=all, D2=V3} in order to get the correct results from
// this non-additive dataset.
private List<String> dimensionsHaveNoPreAggregation = Collections.emptyList();
// The pre-aggregated keyword
private String preAggregatedKeyword = DEFAULT_PREAGGREGATED_DIMENSION_VALUE;
// the actual time duration for non-additive dataset
private Integer nonAdditiveBucketSize;
// the actual time unit for non-additive dataset
private TimeUnit nonAdditiveBucketUnit;
/** End of Configuration for non-additive dataset **/
private boolean realtime = false;
private boolean requiresCompletenessCheck = false;
// delay expected for a dataset for data to arrive
private TimeGranularity expectedDelay = DEFAULT_DAILY_EXPECTED_DELAY;
// algorithm to use for computing data completeness
private DataCompletenessAlgorithmName dataCompletenessAlgorithmName = DataCompletenessAlgorithmName.WO4W_AVERAGE;
// expected percentage completeness for dataset to be marked complete
private double expectedCompleteness = Wo4WAvgDataCompletenessAlgorithm.DEFAULT_EXPECTED_COMPLETENESS;
public String getDataset() {
return dataset;
}
public void setDataset(String dataset) {
this.dataset = dataset;
}
public List<String> getDimensions() {
return dimensions;
}
public void setDimensions(List<String> dimensions) {
this.dimensions = dimensions;
}
public String getTimeColumn() {
return timeColumn;
}
public void setTimeColumn(String timeColumn) {
this.timeColumn = timeColumn;
}
/**
* Use DatasetConfigDTO.bucketTimeGranularity instead of this method for considering the additives of the dataset.
*
* This method is preserved for reading object from database via object mapping (i.e., Java reflection)
*
* @return the time unit of the granularity of the timestamp of each data point.
*/
@Deprecated
public TimeUnit getTimeUnit() {
return timeUnit;
}
public void setTimeUnit(TimeUnit timeUnit) {
this.timeUnit = timeUnit;
}
/**
* Use DatasetConfigDTO.bucketTimeGranularity instead of this method for considering the additives of the dataset.
*
* This method is preserved for reading object from database via object mapping (i.e., Java reflection)
*
* @return the duration of the granularity of the timestamp of each data point.
*/
@Deprecated
public Integer getTimeDuration() {
return timeDuration;
}
public void setTimeDuration(Integer timeDuration) {
this.timeDuration = timeDuration;
}
public String getTimeFormat() {
return timeFormat;
}
public void setTimeFormat(String timeFormat) {
this.timeFormat = timeFormat;
}
public String getTimezone() {
return timezone;
}
public void setTimezone(String timezone) {
this.timezone = timezone;
}
public String getClient() {
return client;
}
public void setClient(String client) {
this.client = client;
}
public boolean isMetricAsDimension() {
return metricAsDimension;
}
public void setMetricAsDimension(boolean metricAsDimension) {
this.metricAsDimension = metricAsDimension;
}
public String getMetricNamesColumn() {
return metricNamesColumn;
}
public void setMetricNamesColumn(String metricNamesColumn) {
this.metricNamesColumn = metricNamesColumn;
}
public String getMetricValuesColumn() {
return metricValuesColumn;
}
public void setMetricValuesColumn(String metricValuesColumn) {
this.metricValuesColumn = metricValuesColumn;
}
public boolean isAutoDiscoverMetrics() {
return autoDiscoverMetrics;
}
public void setAutoDiscoverMetrics(boolean autoDiscoverMetrics) {
this.autoDiscoverMetrics = autoDiscoverMetrics;
}
public boolean isActive() {
return active;
}
public void setActive(boolean active) {
this.active = active;
}
public boolean isAdditive() {
return additive;
}
public void setAdditive(boolean additive) {
this.additive = additive;
}
public List<String> getDimensionsHaveNoPreAggregation() {
return dimensionsHaveNoPreAggregation;
}
public void setDimensionsHaveNoPreAggregation(List<String> dimensionsHaveNoPreAggregation) {
this.dimensionsHaveNoPreAggregation = dimensionsHaveNoPreAggregation;
}
public String getPreAggregatedKeyword() {
return preAggregatedKeyword;
}
public void setPreAggregatedKeyword(String preAggregatedKeyword) {
this.preAggregatedKeyword = preAggregatedKeyword;
}
public Integer getNonAdditiveBucketSize() {
return nonAdditiveBucketSize;
}
public void setNonAdditiveBucketSize(Integer nonAdditiveBucketSize) {
this.nonAdditiveBucketSize = nonAdditiveBucketSize;
}
public TimeUnit getNonAdditiveBucketUnit() {
return nonAdditiveBucketUnit;
}
public void setNonAdditiveBucketUnit(TimeUnit nonAdditiveBucketUnit) {
this.nonAdditiveBucketUnit = nonAdditiveBucketUnit;
}
public boolean isRealtime() {
return realtime;
}
public void setRealtime(boolean realtime) {
this.realtime = realtime;
}
public boolean isRequiresCompletenessCheck() {
return requiresCompletenessCheck;
}
public void setRequiresCompletenessCheck(boolean requiresCompletenessCheck) {
this.requiresCompletenessCheck = requiresCompletenessCheck;
}
public TimeGranularity getExpectedDelay() {
return expectedDelay;
}
public void setExpectedDelay(TimeGranularity expectedDelay) {
this.expectedDelay = expectedDelay;
}
public double getExpectedCompleteness() {
return expectedCompleteness;
}
public void setExpectedCompleteness(double expectedCompleteness) {
this.expectedCompleteness = expectedCompleteness;
}
public DataCompletenessAlgorithmName getDataCompletenessAlgorithmName() {
return dataCompletenessAlgorithmName;
}
public void setDataCompletenessAlgorithmName(
DataCompletenessAlgorithmName dataCompletenessAlgorithmName) {
this.dataCompletenessAlgorithmName = dataCompletenessAlgorithmName;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof DatasetConfigBean)) {
return false;
}
DatasetConfigBean dc = (DatasetConfigBean) o;
return Objects.equals(getId(), dc.getId())
&& Objects.equals(dataset, dc.getDataset())
&& Objects.equals(dimensions, dc.getDimensions())
&& Objects.equals(timeColumn, dc.getTimeColumn())
&& Objects.equals(timeUnit, dc.getTimeUnit())
&& Objects.equals(timeDuration, dc.getTimeDuration())
&& Objects.equals(timeFormat, dc.getTimeFormat())
&& Objects.equals(timezone, dc.getTimezone())
&& Objects.equals(client, dc.getClient())
&& Objects.equals(metricAsDimension, dc.isMetricAsDimension())
&& Objects.equals(metricNamesColumn, dc.getMetricNamesColumn())
&& Objects.equals(metricValuesColumn, dc.getMetricValuesColumn())
&& Objects.equals(autoDiscoverMetrics, dc.isAutoDiscoverMetrics())
&& Objects.equals(active, dc.isActive())
&& Objects.equals(additive, dc.isAdditive())
&& Objects.equals(dimensionsHaveNoPreAggregation, dc.getDimensionsHaveNoPreAggregation())
&& Objects.equals(preAggregatedKeyword, dc.getPreAggregatedKeyword())
&& Objects.equals(nonAdditiveBucketUnit, dc.getNonAdditiveBucketUnit())
&& Objects.equals(nonAdditiveBucketSize, dc.getNonAdditiveBucketSize())
&& Objects.equals(realtime, dc.isRealtime())
&& Objects.equals(requiresCompletenessCheck, dc.isRequiresCompletenessCheck())
&& Objects.equals(expectedDelay, dc.getExpectedDelay())
&& Objects.equals(dataCompletenessAlgorithmName, dc.getDataCompletenessAlgorithmName())
&& Objects.equals(expectedCompleteness,dc.getExpectedCompleteness());
}
@Override
public int hashCode() {
return Objects.hash(getId(), dataset, dimensions, timeColumn, timeUnit, timeDuration, timeFormat, timezone, client,
metricAsDimension, metricNamesColumn, metricValuesColumn, autoDiscoverMetrics, active, additive,
dimensionsHaveNoPreAggregation, preAggregatedKeyword, nonAdditiveBucketSize, nonAdditiveBucketUnit, realtime,
requiresCompletenessCheck, expectedDelay, dataCompletenessAlgorithmName, expectedCompleteness);
}
}