/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.core.startree; import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; import com.clearspring.analytics.stream.cardinality.HyperLogLog; import com.linkedin.pinot.common.data.MetricFieldSpec; import com.linkedin.pinot.common.data.MetricFieldSpec.DerivedMetricType; import com.linkedin.pinot.core.startree.hll.HllUtil; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; /** * fromBytes and toBytes methods are used only in {@link OffHeapStarTreeBuilder}, as read and write to temp files. * Thus no serialization of hll type to string is necessary at these steps. */ public class MetricBuffer { /** * stored as number or hyperLogLog, but serialized out as number or string */ private final Object[] values; private final List<MetricFieldSpec> metricFieldSpecs; public MetricBuffer(Object[] values, List<MetricFieldSpec> metricFieldSpecs) { this.values = values; this.metricFieldSpecs = metricFieldSpecs; } public MetricBuffer(MetricBuffer copy) { this.values = new Object[copy.values.length]; for (int i = 0; i < this.values.length; i++) { Object copyValue = copy.values[i]; if (copyValue instanceof HyperLogLog) { // deep copy of hll field this.values[i] = HllUtil.clone((HyperLogLog)copyValue, HllUtil.getLog2mFromHllFieldSize(copy.metricFieldSpecs.get(i).getFieldSize())); } else if (copyValue instanceof Number) { // number field is immutable this.values[i] = copyValue; } else { throw new IllegalArgumentException("Unsupported metric type: " + copyValue.getClass()); } } this.metricFieldSpecs = copy.metricFieldSpecs; } public static MetricBuffer fromBytes(byte[] bytes, List<MetricFieldSpec> metricFieldSpecs) { ByteBuffer buffer = ByteBuffer.wrap(bytes); Object[] values = new Object[metricFieldSpecs.size()]; for (int i = 0; i < metricFieldSpecs.size(); i++) { MetricFieldSpec metric = metricFieldSpecs.get(i); if (metric.getDerivedMetricType() == DerivedMetricType.HLL) { byte[] hllBytes = new byte[metric.getFieldSize()]; // TODO: buffer reuse buffer.get(hllBytes); values[i] = HllUtil.buildHllFromBytes(hllBytes); } else { switch (metric.getDataType()) { case SHORT: values[i] = buffer.getShort(); break; case INT: values[i] = buffer.getInt(); break; case LONG: values[i] = buffer.getLong(); break; case FLOAT: values[i] = buffer.getFloat(); break; case DOUBLE: values[i] = buffer.getDouble(); break; default: throw new IllegalArgumentException("Unsupported metric type " + metric.getDataType()); } } } return new MetricBuffer(values, metricFieldSpecs); } public byte[] toBytes(int numBytes) throws IOException { byte[] bytes = new byte[numBytes]; ByteBuffer buffer = ByteBuffer.wrap(bytes); for (int i = 0; i < metricFieldSpecs.size(); i++) { MetricFieldSpec metric = metricFieldSpecs.get(i); if (metric.getDerivedMetricType() == DerivedMetricType.HLL) { buffer.put(((HyperLogLog)values[i]).getBytes()); } else { switch (metric.getDataType()) { case SHORT: buffer.putShort(((Number) values[i]).shortValue()); break; case INT: buffer.putInt(((Number) values[i]).intValue()); break; case LONG: buffer.putLong(((Number) values[i]).longValue()); break; case FLOAT: buffer.putFloat(((Number) values[i]).floatValue()); break; case DOUBLE: buffer.putDouble(((Number) values[i]).doubleValue()); break; default: throw new IllegalArgumentException("Unsupported metric type " + metric.getDataType()); } } } return bytes; } public void aggregate(MetricBuffer metrics) { for (int i = 0; i < metricFieldSpecs.size(); i++) { MetricFieldSpec metric = metricFieldSpecs.get(i); if (metric.getDerivedMetricType() == DerivedMetricType.HLL) { try { ((HyperLogLog) values[i]).addAll((HyperLogLog) metrics.values[i]); } catch (CardinalityMergeException e) { throw new RuntimeException(e); } } else { switch (metric.getDataType()) { case SHORT: values[i] = ((Number) values[i]).shortValue() + ((Number) metrics.values[i]).shortValue(); break; case INT: values[i] = ((Number) values[i]).intValue() + ((Number) metrics.values[i]).intValue(); break; case LONG: values[i] = ((Number) values[i]).longValue() + ((Number) metrics.values[i]).longValue(); break; case FLOAT: values[i] = ((Number) values[i]).floatValue() + ((Number) metrics.values[i]).floatValue(); break; case DOUBLE: values[i] = ((Number) values[i]).doubleValue() + ((Number) metrics.values[i]).doubleValue(); break; default: throw new IllegalArgumentException("Unsupported metric type " + metric.getDataType()); } } } } /** * this method should return correct value conformed to datatype to iterators * @param index * @return */ public Object getValueConformToDataType(int index) { if (metricFieldSpecs.get(index).getDerivedMetricType() == DerivedMetricType.HLL) { return HllUtil.convertHllToString((HyperLogLog) values[index]); } else { return values[index]; } } @Override public String toString() { return Arrays.toString(values); } }