/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.addthis.hydra.data.tree.prop;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.math.RoundingMode;
import java.util.regex.Pattern;
import com.addthis.basis.util.ClosableIterator;
import com.addthis.basis.util.Varint;
import com.addthis.bundle.core.BundleField;
import com.addthis.bundle.value.AbstractCustom;
import com.addthis.bundle.value.Numeric;
import com.addthis.bundle.value.ValueArray;
import com.addthis.bundle.value.ValueFactory;
import com.addthis.bundle.value.ValueLong;
import com.addthis.bundle.value.ValueMap;
import com.addthis.bundle.value.ValueObject;
import com.addthis.bundle.value.ValueString;
import com.addthis.bundle.value.ValueTranslationException;
import com.addthis.codec.annotations.FieldConfig;
import com.addthis.codec.codables.BytesCodable;
import com.addthis.hydra.data.tree.DataTreeNode;
import com.addthis.hydra.data.tree.DataTreeNodeUpdater;
import com.addthis.hydra.data.tree.TreeDataParameters;
import com.addthis.hydra.data.tree.TreeNodeData;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.math.DoubleMath;
import org.apache.commons.math3.distribution.GeometricDistribution;
import org.apache.commons.math3.distribution.NormalDistribution;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.Unpooled;
public class DataReservoir extends TreeNodeData<DataReservoir.Config> implements BytesCodable {
private static final Logger log = LoggerFactory.getLogger(DataReservoir.class);
private static final ImmutableList<DataTreeNode> EMPTY_LIST = ImmutableList.<DataTreeNode>builder().build();
private static final byte[] EMPTY_BYTES = new byte[0];
@FieldConfig(codable = true, required = true)
private int[] reservoir;
/**
* The minEpoch is a monotonically increasing value.
* An increase in this value is associated with the elimination
* of state from older epochs. All effort is made to increment the
* value as little as possible.
*/
@FieldConfig(codable = true, required = true)
private long minEpoch;
private BundleField keyAccess;
/**
* This data attachment <span class="hydra-summary">keeps circular buffer
* of N counters</span>.
*
* <p>The numbers of buckets that are stored is determined by the {@link #size}
* parameter. The value stored in the {@link #epochField} bundle field determines
* the current epoch. The counter stored within this epoch is incremented. Older
* epochs are dropped as newer epochs are encountered.
*
* <p>The data attachment can be queried with the notation
* {@code /+%name=epoch=N~percentile=N~obs=N~min=N~mode=modelfit}
* Epoch determines the epoch to be tested. percentile is the Nth percentile
* to use as a threshold. obs specifies how many previous observations to use. All these
* fields are required. Specifying min=N is an optional parameter for a minimum number
* of observations that must be detected. The output returned is of the form
* {@code /delta:+hits/measurement:+hits/mean:+hits/stddev:+hits/mode:+hits/percentile:+hits}.
*
* <p>The data attachment can also be queried with the notation
* {@code /$name=epoch||N~percentile||N~obs||N~min||N~mode||modelfit}.
* The query parameters are identical to those from the previous paragraph
* with the exception that "=" separator has been replaced with the "||" separator
* (sideways equals?). The output returned is a value array with six elements.
* The contents of the array are ["delta", "measurement", "mean", "stddev", "mode", "percentile"].
*
* @user-reference
*/
public static final class Config extends TreeDataParameters<DataReservoir> {
/**
* Bundle field name from which to draw the epoch.
* This field is required.
*/
@FieldConfig(codable = true, required = true)
private String epochField;
/**
* Size of the reservoir. This field is required.
* @return
*/
@FieldConfig(codable = true, required = true)
private int size;
@Override
public DataReservoir newInstance() {
return new DataReservoir();
}
}
public DataReservoir() {
}
public DataReservoir(DataReservoir other) {
reservoir = (other.reservoir == null) ? null : other.reservoir.clone();
minEpoch = other.minEpoch;
}
public DataReservoir(long minEpoch, int[] reservoir) {
this.minEpoch = minEpoch;
this.reservoir = reservoir;
}
/**
* Resize the reservoir to the new size.
* If the reservoir has not yet been allocated then
* construct it. If the requested length is smaller
* than the reservoir then discard the oldest values.
* If the requested length is larger than the reservoir
* then allocate additional space for the reservoir.
*
* @param newsize new size of the reservoir
*/
private void resize(int newsize) {
if (reservoir == null) {
reservoir = new int[newsize];
} else if (reservoir.length < newsize) {
int[] newReservoir = new int[newsize];
System.arraycopy(reservoir, 0, newReservoir, 0, reservoir.length);
reservoir = newReservoir;
} else if (reservoir.length > newsize) {
int[] newReservoir = new int[newsize];
System.arraycopy(reservoir, reservoir.length - newsize, newReservoir, 0, newsize);
minEpoch += (reservoir.length - newsize);
reservoir = newReservoir;
}
}
/**
* Shift the minimum epoch to accommodate
* the new epoch. If the epoch is less than the minimum
* epoch then do nothing. If the epoch falls within the boundary
* of the the reservoir then do nothing. If the epoch is farther
* away then one length away from the maximum epoch, then empty
* out the reservoir and set the maximum epoch to the target
* epoch. Otherwise shift the reservoir to accommodate the new
* epoch.
*
* @param epoch new epoch to accommodate
*/
private void shift(long epoch) {
long delta = (epoch - minEpoch);
if (delta < reservoir.length) {
// do nothing
} else if (delta > 2 * (reservoir.length - 1)) {
Arrays.fill(reservoir, 0);
minEpoch = epoch - (reservoir.length - 1);
} else {
int shift = (int) (delta - reservoir.length + 1);
System.arraycopy(reservoir, shift, reservoir, 0, reservoir.length - shift);
Arrays.fill(reservoir, reservoir.length - shift, reservoir.length, 0);
minEpoch += shift;
}
}
/**
* Insert the new epoch. Assumes that {@link #shift(long epoch)}
* has previously been invoked.
*
* @param epoch new epoch to insert
*/
private void update(long epoch, long count) {
if (epoch >= minEpoch) {
reservoir[(int) (epoch - minEpoch)] += count;
}
}
public DataReservoir merge(DataReservoir other) {
if (this.reservoir == null) {
return new DataReservoir(other);
} else if (other.reservoir == null) {
return new DataReservoir(this);
}
long minEpoch1 = this.minEpoch;
long minEpoch2 = other.minEpoch;
long maxEpoch1 = this.minEpoch + this.reservoir.length;
long maxEpoch2 = other.minEpoch + other.reservoir.length;
long minEpoch = Math.min(minEpoch1, minEpoch2);
long maxEpoch = Math.max(maxEpoch1, maxEpoch2);
DataReservoir result = new DataReservoir();
result.minEpoch = minEpoch;
result.resize((int) (maxEpoch - minEpoch));
for (long i = minEpoch; i < maxEpoch; i++) {
long count1 = Math.max(0, this.retrieveCount(i));
long count2 = Math.max(0, other.retrieveCount(i));
result.update(i, count1 + count2);
}
return result;
}
/**
* Update the reservoir with the input epoch and a value of one.
*
* @param epoch input time period
* @param size alters the capacity of the reservoir
*/
@VisibleForTesting
void updateReservoir(long epoch, int size) {
updateReservoir(epoch, size, 1);
}
/**
* Update the reservoir with the input epoch and specified additional count.
*
* @param epoch input time period
* @param size alters the capacity of the reservoir
* @param count amount to increment the time period
*/
@VisibleForTesting
void updateReservoir(long epoch, int size, long count) {
resize(size);
shift(epoch);
update(epoch, count);
}
/**
* Return the count associated with the input epoch,
* or an error value if the input is out of bounds.
*
* @param epoch target epoch
* @return the non-negative count or -1 if input is less
* than minimum epoch or -2 if input is greater
* than maximum epoch or -3 if the data structure
* has not been initialized.
*/
@VisibleForTesting
int retrieveCount(long epoch) {
if (reservoir == null) {
return -3;
} else if (epoch < minEpoch) {
return -1;
} else if (epoch >= (minEpoch + reservoir.length)) {
return -2;
} else {
return reservoir[(int) (epoch - minEpoch)];
}
}
@Override
public boolean updateChildData(DataTreeNodeUpdater state,
DataTreeNode childNode, DataReservoir.Config conf) {
if (keyAccess == null) {
keyAccess = state.getBundle().getFormat().getField(conf.epochField);
}
ValueObject val = state.getBundle().getValue(keyAccess);
if (val != null) {
try {
long epoch = val.asLong().getLong();
updateReservoir(epoch, conf.size);
return true;
} catch (Exception ex) {
log.error("Error trying to insert " + val + " into reservoir: ", ex);
}
}
return false;
}
/**
* Helper method for {@link #getNodes(com.addthis.hydra.data.tree.DataTreeNode, String)}
* If raw=true then add nodes for the raw observations.
*/
private void addRawObservations(List<DataTreeNode> result, long targetEpoch, int numObservations) {
if (targetEpoch < 0 || targetEpoch >= minEpoch + reservoir.length) {
targetEpoch = minEpoch + reservoir.length - 1;
}
if (numObservations < 0 || numObservations > reservoir.length - 1) {
numObservations = reservoir.length - 1;
}
int count = 0;
int index = reservoir.length - 1;
long currentEpoch = minEpoch + index;
while (currentEpoch != targetEpoch) {
index--;
currentEpoch--;
}
/**
* numObservations elements for the historical value.
* Add one element to store for the target epoch.
* Add one element to store the "minEpoch" node.
*/
VirtualTreeNode[] children = new VirtualTreeNode[numObservations + 2];
children[count++] = new VirtualTreeNode(Long.toString(currentEpoch), reservoir[index--]);
while (count <= numObservations && index >= 0) {
children[count++] = new VirtualTreeNode(Long.toString(minEpoch + index), reservoir[index--]);
}
while (count <= numObservations) {
children[count++] = new VirtualTreeNode(Long.toString(minEpoch + index), 0);
index--;
}
children[count] = new VirtualTreeNode("minEpoch", minEpoch);
result.add(new VirtualTreeNode("observations", 1, children));
}
/**
* Either generate some nodes for debugging purposes or
* return an empty list.
*
* @param raw if true then generate debugging nodes
* @return list of nodes
*/
private List<DataTreeNode> makeDefaultNodes(boolean raw, long targetEpoch, int numObservations) {
if (raw) {
List<DataTreeNode> result = new ArrayList<>();
addRawObservations(result, targetEpoch, numObservations);
return result;
} else {
return EMPTY_LIST;
}
}
/**
* Convenience method to convert an node into an array of size one.
*/
private static VirtualTreeNode[] generateSingletonArray(VirtualTreeNode value) {
VirtualTreeNode[] result = new VirtualTreeNode[1];
result[0] = value;
return result;
}
private static double longToDouble(long value, boolean doubleToLongBits) {
if (doubleToLongBits) {
return Double.longBitsToDouble(value);
} else {
return (double) value;
}
}
private static long doubleToLong(double value, boolean doubleToLongBits) {
if (doubleToLongBits) {
return Double.doubleToLongBits(value);
} else {
return DoubleMath.roundToLong(value, RoundingMode.HALF_UP);
}
}
private ValueObject generateValueObject(String key, String separator) {
long targetEpoch = -1;
int numObservations = -1;
double sigma = Double.POSITIVE_INFINITY;
double percentile = 0;
boolean doubleToLongBits = false;
int minMeasurement = Integer.MIN_VALUE;
boolean raw = false;
String mode = "sigma";
if (key == null) {
return null;
}
String[] kvpairs = key.split(Pattern.quote("~"));
for(String kvpair : kvpairs) {
String[] kv = kvpair.split(Pattern.quote(separator));
if (kv.length == 2) {
String kvkey = kv[0];
String kvvalue = kv[1];
switch (kvkey) {
case "double":
doubleToLongBits = Boolean.parseBoolean(kvvalue);
break;
case "epoch":
targetEpoch = Long.parseLong(kvvalue);
break;
case "sigma":
sigma = Double.parseDouble(kvvalue);
break;
case "min":
minMeasurement = Integer.parseInt(kvvalue);
break;
case "obs":
numObservations = Integer.parseInt(kvvalue);
break;
case "raw":
raw = Boolean.parseBoolean(kvvalue);
break;
case "percentile":
percentile = Double.parseDouble(kvvalue);
break;
case "mode":
mode = kvvalue;
break;
default:
throw new RuntimeException("Unknown key " + kvkey);
}
}
}
if (mode.equals("get")) {
long count = retrieveCount(targetEpoch);
if (count < 0) {
return ValueFactory.create(0L);
} else {
return ValueFactory.create(count);
}
} else {
DataReservoirValue.Builder builder = new DataReservoirValue.Builder();
builder.setTargetEpoch(targetEpoch);
builder.setNumObservations(numObservations);
builder.setDoubleToLongBits(doubleToLongBits);
builder.setRaw(raw);
builder.setSigma(sigma);
builder.setMinMeasurement(minMeasurement);
builder.setPercentile(percentile);
builder.setMode(mode);
DataReservoirValue value = builder.build(this);
return value;
}
}
private List<DataTreeNode> computeResult(DataReservoirValue value) {
switch (value.mode) {
case "sigma":
return sigmaAnomalyDetection(value.targetEpoch, value.numObservations, value.doubleToLongBits,
value.raw, value.sigma, value.minMeasurement);
case "modelfit":
return modelFitAnomalyDetection(value.targetEpoch, value.numObservations, value.doubleToLongBits,
value.raw, value.percentile, value.minMeasurement);
default:
throw new RuntimeException("Unknown mode type '" + value.mode + "'");
}
}
@Override
public ValueObject getValue(String key) {
if (key == null) {
return null;
}
ValueObject value = generateValueObject(key, "||");
if (value instanceof DataReservoirValue) {
DataReservoirValue dataReservoirValue = (DataReservoirValue) value;
value = dataReservoirValue.setDoubleToLongClone(true).setRawClone(false);
}
return value;
}
@Override
public List<DataTreeNode> getNodes(DataTreeNode parent, String key) {
if (key == null) {
return null;
}
DataReservoirValue value = (DataReservoirValue) generateValueObject(key, "=");
return computeResult(value);
}
private static void updateFrequencies(Map<Integer,Integer> frequencies, int value) {
Integer count = frequencies.get(value);
if (count == null) {
count = 0;
}
frequencies.put(value, count + 1);
}
private double gaussianNegativeProbability(double mean, double stddev) {
NormalDistribution distribution = new NormalDistribution(mean, stddev);
return distribution.cumulativeProbability(0.0);
}
@VisibleForTesting
List<DataTreeNode> modelFitAnomalyDetection(long targetEpoch, int numObservations,
boolean doubleToLongBits, boolean raw, double percentile, int minMeasurement) {
int measurement;
int count = 0;
int min = Integer.MAX_VALUE;
if (targetEpoch < 0) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (numObservations <= 0) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (reservoir == null) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (targetEpoch < minEpoch) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (targetEpoch >= minEpoch + reservoir.length) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (numObservations > (reservoir.length - 1)) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
}
/**
* Fitting to a geometric distribution uses the mean value of the sample.
*
* Fitting to a normal distribution uses the Apache Commons Math implementation.
*/
double mean = 0.0;
double m2 = 0.0;
double stddev;
double gaussianNegative = -1.0;
Map<Integer,Integer> frequencies = new HashMap<>();
double threshold;
double measurePercentile = -100.0;
int index = reservoir.length - 1;
long currentEpoch = minEpoch + index;
while (currentEpoch != targetEpoch) {
index--;
currentEpoch--;
}
measurement = reservoir[index--];
currentEpoch--;
while (count < numObservations && index >= 0) {
int value = reservoir[index--];
if (value < min) {
min = value;
}
updateFrequencies(frequencies, value);
count++;
double delta = value - mean;
mean += delta / count;
m2 += delta * (value - mean);
}
while (count < numObservations) {
int value = 0;
if (value < min) {
min = value;
}
updateFrequencies(frequencies, value);
count++;
double delta = value - mean;
mean += delta / count;
m2 += delta * (value - mean);
}
if (count < 2) {
stddev = 0.0;
} else {
stddev = Math.sqrt(m2 / count);
}
int mode = -1;
int modeCount = -1;
for(Map.Entry<Integer,Integer> entry : frequencies.entrySet()) {
int key = entry.getKey();
int value = entry.getValue();
if (value > modeCount || (value == modeCount && key > mode)) {
mode = key;
modeCount = value;
}
}
if (mean > 0.0 && stddev > 0.0) {
gaussianNegative = gaussianNegativeProbability(mean, stddev);
}
if (mean == 0.0) {
threshold = 0.0;
} else if (stddev == 0.0) {
threshold = mean;
} else if (mean > 1.0) {
NormalDistribution distribution = new NormalDistribution(mean, stddev);
double badProbability = distribution.cumulativeProbability(0.0);
double goodProbability = badProbability + (1.0 - badProbability) * (percentile / 100.0);
threshold = distribution.inverseCumulativeProbability(goodProbability);
measurePercentile = distribution.probability(0.0, measurement) / (1.0 - badProbability) * 100.0;
} else {
double p = 1.0 / (1.0 + mean);
GeometricDistribution distribution = new GeometricDistribution(p);
threshold = distribution.inverseCumulativeProbability(percentile / 100.0);
measurePercentile = distribution.cumulativeProbability(measurement) * 100.0;
}
List<DataTreeNode> result = new ArrayList<>();
VirtualTreeNode vchild, vparent;
if (measurement >= minMeasurement && (measurement > threshold || percentile == 0.0)) {
vchild = new VirtualTreeNode("gaussianNegative",
doubleToLong(gaussianNegative, doubleToLongBits));
vparent = new VirtualTreeNode("percentile",
doubleToLong(measurePercentile, doubleToLongBits), generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("mode", mode, generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("stddev",
doubleToLong(stddev, doubleToLongBits), generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("mean",
doubleToLong(mean, doubleToLongBits), generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("measurement",
measurement, generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("delta",
doubleToLong(measurement - threshold, doubleToLongBits), generateSingletonArray(vchild));
result.add(vparent);
if (raw) {
addRawObservations(result, targetEpoch, numObservations);
}
} else {
makeDefaultNodes(raw, targetEpoch, numObservations);
}
return result;
}
private List<DataTreeNode> sigmaAnomalyDetection(long targetEpoch, int numObservations, boolean doubleToLongBits,
boolean raw, double sigma, int minMeasurement) {
int measurement;
if (targetEpoch < 0) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (sigma == Double.POSITIVE_INFINITY) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (numObservations <= 0) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (reservoir == null) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (targetEpoch < minEpoch) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (targetEpoch >= minEpoch + reservoir.length) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
} else if (numObservations > (reservoir.length - 1)) {
return makeDefaultNodes(raw, targetEpoch, numObservations);
}
int count = 0;
double mean = 0.0;
double m2 = 0.0;
double stddev;
int index = reservoir.length - 1;
long currentEpoch = minEpoch + index;
while (currentEpoch != targetEpoch) {
index--;
currentEpoch--;
}
measurement = reservoir[index--];
while (count < numObservations && index >= 0) {
int value = reservoir[index--];
count++;
double delta = value - mean;
mean += delta / count;
m2 += delta * (value - mean);
}
while (count < numObservations) {
int value = 0;
count++;
double delta = value - mean;
mean += delta / count;
m2 += delta * (value - mean);
}
if (count < 2) {
stddev = 0.0;
} else {
stddev = Math.sqrt(m2 / count);
}
double delta = (measurement - (sigma * stddev + mean));
VirtualTreeNode vchild, vparent;
if (delta >= 0 && measurement >= minMeasurement) {
List<DataTreeNode> result = new ArrayList<>();
vchild = new VirtualTreeNode("threshold",
doubleToLong(sigma * stddev + mean, doubleToLongBits));
vparent = new VirtualTreeNode("stddev",
doubleToLong(stddev, doubleToLongBits), generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("mean",
doubleToLong(mean, doubleToLongBits), generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("measurement",
measurement, generateSingletonArray(vchild));
vchild = vparent;
vparent = new VirtualTreeNode("delta",
doubleToLong(delta, doubleToLongBits), generateSingletonArray(vchild));
result.add(vparent);
if (raw) {
addRawObservations(result, targetEpoch, numObservations);
}
return result;
} else {
return makeDefaultNodes(raw, targetEpoch, numObservations);
}
}
static final class DataReservoirValue extends AbstractCustom<DataReservoir>
implements Numeric {
long targetEpoch;
int numObservations;
boolean doubleToLongBits;
boolean raw;
double percentile;
double sigma;
int minMeasurement;
String mode;
/**
* The Builder pattern allows many different variations of a class to
* be instantiated without the pitfalls of complex constructors. See
* ''Effective Java, Second Edition.'' Item 2 - "Consider a builder when
* faced with many constructor parameters."
*/
static class Builder {
long targetEpoch = -1;
int numObservations = -1;
boolean doubleToLongBits = false;
boolean raw = false;
double percentile = 0;
double sigma = Double.POSITIVE_INFINITY;
int minMeasurement = Integer.MIN_VALUE;
String mode = "sigma";
Builder setTargetEpoch(long targetEpoch) {
this.targetEpoch = targetEpoch;
return this;
}
Builder setNumObservations(int numObservations) {
this.numObservations = numObservations;
return this;
}
Builder setDoubleToLongBits(boolean doubleToLongBits) {
this.doubleToLongBits = doubleToLongBits;
return this;
}
Builder setRaw(boolean raw) {
this.raw = raw;
return this;
}
Builder setPercentile(double percentile) {
this.percentile = percentile;
return this;
}
Builder setSigma(double sigma) {
this.sigma = sigma;
return this;
}
Builder setMinMeasurement(int minMeasurement) {
this.minMeasurement = minMeasurement;
return this;
}
Builder setMode(String mode) {
this.mode = mode;
return this;
}
DataReservoirValue build(DataReservoir reservoir) {
return new DataReservoirValue(reservoir, targetEpoch, numObservations,
doubleToLongBits, raw, percentile, sigma, minMeasurement, mode);
}
}
@SuppressWarnings("unused")
public DataReservoirValue() {
super(null);
}
private DataReservoirValue(DataReservoir reservoir, long targetEpoch, int numObservations,
boolean doubleToLongBits, boolean raw, double percentile, double sigma,
int minMeasurement, String mode) {
super(reservoir);
this.targetEpoch = targetEpoch;
this.numObservations = numObservations;
this.doubleToLongBits = doubleToLongBits;
this.raw = raw;
this.percentile = percentile;
this.sigma = sigma;
this.minMeasurement = minMeasurement;
this.mode = mode;
}
public DataReservoirValue setRawClone(boolean raw) {
return new DataReservoirValue(asNative(), targetEpoch, numObservations,
doubleToLongBits, raw, percentile, sigma, minMeasurement, mode);
}
public DataReservoirValue setDoubleToLongClone(boolean doubleToLongBits) {
return new DataReservoirValue(asNative(), targetEpoch, numObservations,
doubleToLongBits, raw, percentile, sigma, minMeasurement, mode);
}
@Override
public Numeric sum(Numeric val) {
return new DataReservoirValue(asNative().merge((DataReservoir) val.asNative()),
targetEpoch, numObservations, doubleToLongBits, raw, percentile,
sigma, minMeasurement, mode);
}
@Override
public Numeric diff(Numeric val) {
throw new UnsupportedOperationException();
}
@Override
public Numeric prod(Numeric val) {
throw new UnsupportedOperationException();
}
@Override
public Numeric divide(Numeric val) {
throw new UnsupportedOperationException();
}
@Override
public Numeric avg(int count) {
throw new UnsupportedOperationException();
}
@Override
public Numeric min(Numeric val) {
throw new UnsupportedOperationException();
}
@Override
public Numeric max(Numeric val) {
throw new UnsupportedOperationException();
}
@Override
public ValueMap asMap() {
ValueMap result = ValueFactory.createMap();
result.put("targetEpoch", ValueFactory.create(targetEpoch));
result.put("numObservations", ValueFactory.create(numObservations));
result.put("doubleToLongBits", ValueFactory.create(doubleToLongBits ? 1 : 0));
result.put("raw", ValueFactory.create(raw ? 1 : 0));
result.put("percentile", ValueFactory.create(percentile));
result.put("sigma", ValueFactory.create(sigma));
result.put("minMeasurement", ValueFactory.create(minMeasurement));
result.put("mode", ValueFactory.create(mode));
result.put("minEpoch", ValueFactory.create(heldObject.minEpoch));
ValueArray reservoir = ValueFactory.createArray(heldObject.reservoir.length);
for (int i = 0; i < heldObject.reservoir.length; i++) {
reservoir.add(i, ValueFactory.create(heldObject.reservoir[i]));
}
result.put("reservoir", reservoir);
return result;
}
@Override
public void setValues(ValueMap map) {
targetEpoch = map.get("targetEpoch").asLong().asNative();
numObservations = map.get("numObservations").asLong().asNative().intValue();
doubleToLongBits = map.get("doubleToLongBits").asLong().asNative().intValue() == 1;
raw = map.get("raw").asLong().asNative().intValue() == 1;
percentile = map.get("percentile").asDouble().asNative();
sigma = map.get("sigma").asDouble().asNative();
minMeasurement = map.get("minMeasurement").asLong().asNative().intValue();
mode = map.get("mode").asString().asNative();
long minEpoch = map.get("minEpoch").asLong().asNative();
ValueArray reservoirValueObject = map.get("reservoir").asArray();
int size = reservoirValueObject.size();
int[] reservoir = new int[size];
for (int i = 0; i < size; i++) {
reservoir[i] = reservoirValueObject.get(i).asLong().asNative().intValue();
}
this.heldObject = new DataReservoir(minEpoch, reservoir);
}
@Override
public ValueLong asLong() { return ValueFactory.create(asArray().size()); }
@Override
public ValueArray asArray() throws ValueTranslationException {
ValueArray result = ValueFactory.createArray(1);
List<DataTreeNode> list = asNative().computeResult(this);
ClosableIterator<DataTreeNode> iterator;
DataTreeNode current;
switch (mode) {
case "modelfit":
current = list.get(0);
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'delta'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(current.getCounter())); // 'measurement'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'mean'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'stddev'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(current.getCounter())); // 'mode'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'percentile'
break;
case "sigma":
current = list.get(0);
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'delta'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(current.getCounter())); // 'measurement'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'mean'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'stddev'
iterator = current.getIterator();
current = iterator.next();
iterator.close();
result.add(ValueFactory.create(longToDouble(current.getCounter(), doubleToLongBits))); // 'threshold'
break;
default:
throw new RuntimeException("Unknown mode type '" + mode + "'");
}
return result;
}
@Override
public ValueString asString() throws ValueTranslationException {
throw new ValueTranslationException();
}
}
@Override
public byte[] bytesEncode(long version) {
if (reservoir == null) {
return EMPTY_BYTES;
}
byte[] retBytes = null;
ByteBuf byteBuf = PooledByteBufAllocator.DEFAULT.buffer();
try {
Varint.writeUnsignedVarLong(minEpoch, byteBuf);
Varint.writeUnsignedVarInt(reservoir.length, byteBuf);
for(int element : reservoir) {
Varint.writeUnsignedVarInt(element, byteBuf);
}
retBytes = new byte[byteBuf.readableBytes()];
byteBuf.readBytes(retBytes);
} finally {
byteBuf.release();
}
return retBytes;
}
@Override
public void bytesDecode(byte[] b, long version) {
if (b.length == 0) {
return;
}
ByteBuf byteBuf = Unpooled.wrappedBuffer(b);
try {
minEpoch = Varint.readUnsignedVarLong(byteBuf);
int length = Varint.readUnsignedVarInt(byteBuf);
reservoir = new int[length];
for (int i = 0; i < reservoir.length; i++) {
reservoir[i] = Varint.readUnsignedVarInt(byteBuf);
}
} finally {
byteBuf.release();
}
}
}