/*
* Copyright © 2014-2015 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package co.cask.cdap.metrics.query;
import co.cask.cdap.api.dataset.lib.cube.AggregationFunction;
import co.cask.cdap.api.dataset.lib.cube.TimeValue;
import co.cask.cdap.api.metrics.MetricDataQuery;
import co.cask.cdap.api.metrics.MetricStore;
import co.cask.cdap.api.metrics.MetricTimeSeries;
import co.cask.cdap.common.conf.Constants;
import co.cask.cdap.common.queue.QueueName;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
import com.google.common.collect.PeekingIterator;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
/**
* Executes metrics requests, returning a json object representing the result of the request.
*/
// todo: remove it when v2/metrics APIs are gone
public class MetricStoreRequestExecutor {
private static final Logger LOG = LoggerFactory.getLogger(MetricStoreRequestExecutor.class);
private static final Gson GSON = new Gson();
private final MetricStore metricStore;
public MetricStoreRequestExecutor(final MetricStore metricStore) {
this.metricStore = metricStore;
}
public JsonElement executeQuery(MetricDataQuery query) throws Exception {
// Pretty ugly logic now. Need to refactor
Object resultObj;
if (query.getResolution() != Integer.MAX_VALUE) {
TimeSeriesResponse.Builder builder = TimeSeriesResponse.builder(query.getStartTs(),
query.getEndTs());
// Special metrics handle that requires computation from multiple time series.
if (query.getMetrics().containsKey("system.process.busyness")) {
computeProcessBusyness(query, builder);
} else {
PeekingIterator<TimeValue> timeValueItor = Iterators.peekingIterator(queryTimeSeries(query));
long resultTimeStamp = (query.getStartTs() / query.getResolution()) * query.getResolution();
for (int i = 0; i < query.getLimit(); i++) {
if (timeValueItor.hasNext() && timeValueItor.peek().getTimestamp() == resultTimeStamp) {
builder.addData(resultTimeStamp, timeValueItor.next().getValue());
} else {
// If the scan result doesn't have value for a timestamp, we add 0 to the result-returned for that timestamp
builder.addData(resultTimeStamp, 0);
}
resultTimeStamp += query.getResolution();
}
}
resultObj = builder.build();
} else {
// Special metrics handle that requires computation from multiple aggregates results.
if (query.getMetrics().containsKey("system.process.events.pending")) {
resultObj = computeFlowletPending(query);
} else {
resultObj = getAggregates(query);
}
}
return GSON.toJsonTree(resultObj);
}
private void computeProcessBusyness(MetricDataQuery query, TimeSeriesResponse.Builder builder) throws Exception {
PeekingIterator<TimeValue> tuplesReadItor =
Iterators.peekingIterator(queryTimeSeries(new MetricDataQuery(query, "system.process.tuples.read",
AggregationFunction.SUM)));
PeekingIterator<TimeValue> eventsProcessedItor =
Iterators.peekingIterator(queryTimeSeries(new MetricDataQuery(query, "system.process.events.processed",
AggregationFunction.SUM)));
long resultTimeStamp = query.getStartTs();
for (int i = 0; i < query.getLimit(); i++) {
long tupleRead = 0;
long eventProcessed = 0;
if (tuplesReadItor.hasNext() && tuplesReadItor.peek().getTimestamp() == resultTimeStamp) {
tupleRead = tuplesReadItor.next().getValue();
}
if (eventsProcessedItor.hasNext() && eventsProcessedItor.peek().getTimestamp() == resultTimeStamp) {
eventProcessed = eventsProcessedItor.next().getValue();
}
if (eventProcessed != 0) {
int busyness = (int) ((float) tupleRead / eventProcessed * 100);
builder.addData(resultTimeStamp, busyness > 100 ? 100 : busyness);
} else {
// If the scan result doesn't have value for a timestamp, we add 0 to the returned result for that timestamp.
builder.addData(resultTimeStamp, 0);
}
resultTimeStamp += query.getResolution();
}
}
private Object computeFlowletPending(MetricDataQuery query) throws Exception {
// Pending is processed by flowlet minus emitted into the queues it was processing from.
// trick: get all queues and streams it was processing from using group by
MetricDataQuery groupByQueueName =
new MetricDataQuery(new MetricDataQuery(query, "system.process.events.processed", AggregationFunction.SUM),
ImmutableList.of(Constants.Metrics.Tag.FLOWLET_QUEUE));
Map<String, Long> processedPerQueue = getTotalsWithSingleGroupByTag(groupByQueueName);
long processedTotal = 0;
long writtenTotal = 0;
for (Map.Entry<String, Long> entry : processedPerQueue.entrySet()) {
String name = entry.getKey();
// note: each has "input." prefix
QueueName queueName = QueueName.from(URI.create(name.substring("input.".length(), name.length())));
long written;
if (queueName.isQueue()) {
Map<String, String> sliceByTags = Maps.newHashMap(query.getSliceByTags());
// we want to aggregate written to the queue by all flowlets
sliceByTags.remove(Constants.Metrics.Tag.FLOWLET);
// we want to narrow down to specific queue we know our flowlet was consuming from
sliceByTags.put(Constants.Metrics.Tag.FLOWLET_QUEUE, queueName.getSimpleName());
written = getTotals(new MetricDataQuery(new MetricDataQuery(query, sliceByTags),
"system.process.events.out", AggregationFunction.SUM));
} else if (queueName.isStream()) {
Map<String, String> sliceByTags = Maps.newHashMap();
// we want to narrow down to specific stream we know our flowlet was consuming from
sliceByTags.put(Constants.Metrics.Tag.STREAM, queueName.getSimpleName());
// note: namespace + stream uniquely define the stream
// we know that flow can consume from stream of the same namespace only at this point
sliceByTags.put(Constants.Metrics.Tag.NAMESPACE, query.getSliceByTags().get(Constants.Metrics.Tag.NAMESPACE));
written = getTotals(new MetricDataQuery(new MetricDataQuery(query, sliceByTags),
"system.collect.events", AggregationFunction.SUM));
} else {
LOG.warn("Unknown queue type: " + name);
continue;
}
processedTotal += entry.getValue();
writtenTotal += written;
}
long pending = writtenTotal - processedTotal;
return new AggregateResponse(pending > 0 ? pending : 0);
}
private Iterator<TimeValue> queryTimeSeries(MetricDataQuery query) throws Exception {
Collection<MetricTimeSeries> result = metricStore.query(query);
if (result.size() == 0) {
return new ArrayList<TimeValue>().iterator();
}
// since there's no group by condition, it'll return single time series always
MetricTimeSeries timeSeries = result.iterator().next();
return Iterables.transform(timeSeries.getTimeValues(),
new Function<TimeValue, TimeValue>() {
@Override
public TimeValue apply(TimeValue input) {
return new TimeValue(input.getTimestamp(), input.getValue());
}
}).iterator();
}
private AggregateResponse getAggregates(MetricDataQuery query) throws Exception {
return new AggregateResponse(getTotals(query));
}
private long getTotals(MetricDataQuery query) throws Exception {
// query must have resolution set to Integer.MAX_VALUE (i.e. "totals")
Collection<MetricTimeSeries> result = metricStore.query(query);
if (result.size() == 0) {
return 0;
}
// since there's no group by condition, it'll return single time series always
MetricTimeSeries timeSeries = result.iterator().next();
if (timeSeries.getTimeValues().isEmpty()) {
return 0;
}
// since it is totals, it will have only one TimeValue or none
return timeSeries.getTimeValues().get(0).getValue();
}
private Map<String, Long> getTotalsWithSingleGroupByTag(MetricDataQuery query) throws Exception {
// query must have resolution set to Integer.MAX_VALUE (i.e. "totals")
Collection<MetricTimeSeries> result = metricStore.query(query);
Map<String, Long> map = Maps.newHashMap();
for (MetricTimeSeries timeSeries : result) {
// we know there's only ony group by tag
String groupByTagValue = timeSeries.getTagValues().values().iterator().next();
// since it is totals, it will have only one TimeValue
map.put(groupByTagValue, timeSeries.getTimeValues().get(0).getValue());
}
return map;
}
}