/* PerformanceMonitor.java
*
* Copyright 2009-2012 Comcast Interactive Media, LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.fishwife.jrugged;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
/**
* The {@link PerformanceMonitor} is a convenience wrapper for gathering a slew of useful
* operational metrics about a service, including moving averages for latency and request
* rate over various time windows (last minute, last hour, last day).
*
* The intended use is for a client to use the "Decorator" design pattern that decorates
* an existing service with this wrapper. Portions of this object can then be exposed via
* JMX, for example to allow for operational polling.
*/
public class PerformanceMonitor implements ServiceWrapper {
private static final String WRAP_MSG = "org.fishwife.jrugged.PerformanceMonitor.WRAPPED";
private final long startupMillis = System.currentTimeMillis();
private static final long ONE_MINUTE_MILLIS = 60L * 1000L;
private static final long ONE_HOUR_MILLIS = ONE_MINUTE_MILLIS * 60L;
private static final long ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24L;
private final RequestCounter requestCounter = new RequestCounter();
private final FlowMeter flowMeter = new FlowMeter(requestCounter);
private MovingAverage averageSuccessLatencyLastMinute;
private MovingAverage averageSuccessLatencyLastHour;
private MovingAverage averageSuccessLatencyLastDay;
private MovingAverage averageFailureLatencyLastMinute;
private MovingAverage averageFailureLatencyLastHour;
private MovingAverage averageFailureLatencyLastDay;
private MovingAverage totalRequestsPerSecondLastMinute;
private MovingAverage successRequestsPerSecondLastMinute;
private MovingAverage failureRequestsPerSecondLastMinute;
private MovingAverage totalRequestsPerSecondLastHour;
private MovingAverage successRequestsPerSecondLastHour;
private MovingAverage failureRequestsPerSecondLastHour;
private MovingAverage totalRequestsPerSecondLastDay;
private MovingAverage successRequestsPerSecondLastDay;
private MovingAverage failureRequestsPerSecondLastDay;
private final SampledQuantile lifetimeSuccessLatencyQuantile = new SampledQuantile();
private final SampledQuantile lifetimeFailureLatencyQuantile = new SampledQuantile();
private final SampledQuantile successLatencyQuantileLastMinute = new SampledQuantile(
60L, TimeUnit.SECONDS);
private final SampledQuantile successLatencyQuantileLastHour = new SampledQuantile(
3600L, TimeUnit.SECONDS);
private final SampledQuantile successLatencyQuantileLastDay = new SampledQuantile(
86400L, TimeUnit.SECONDS);
private final SampledQuantile failureLatencyQuantileLastMinute = new SampledQuantile(
60L, TimeUnit.SECONDS);
private final SampledQuantile failureLatencyQuantileLastHour = new SampledQuantile(
3600L, TimeUnit.SECONDS);
private final SampledQuantile failureLatencyQuantileLastDay = new SampledQuantile(
86400L, TimeUnit.SECONDS);
private long lifetimeMaxSuccessMillis;
private long lifetimeMaxFailureMillis;
/** Default constructor. */
public PerformanceMonitor() {
createMovingAverages();
}
private void createMovingAverages() {
averageSuccessLatencyLastMinute = new MovingAverage(ONE_MINUTE_MILLIS);
averageSuccessLatencyLastHour = new MovingAverage(ONE_HOUR_MILLIS);
averageSuccessLatencyLastDay = new MovingAverage(ONE_DAY_MILLIS);
averageFailureLatencyLastMinute = new MovingAverage(ONE_MINUTE_MILLIS);
averageFailureLatencyLastHour = new MovingAverage(ONE_HOUR_MILLIS);
averageFailureLatencyLastDay = new MovingAverage(ONE_DAY_MILLIS);
totalRequestsPerSecondLastMinute = new MovingAverage(ONE_MINUTE_MILLIS);
successRequestsPerSecondLastMinute = new MovingAverage(ONE_MINUTE_MILLIS);
failureRequestsPerSecondLastMinute = new MovingAverage(ONE_MINUTE_MILLIS);
totalRequestsPerSecondLastHour = new MovingAverage(ONE_HOUR_MILLIS);
successRequestsPerSecondLastHour = new MovingAverage(ONE_HOUR_MILLIS);
failureRequestsPerSecondLastHour = new MovingAverage(ONE_HOUR_MILLIS);
totalRequestsPerSecondLastDay = new MovingAverage(ONE_DAY_MILLIS);
successRequestsPerSecondLastDay = new MovingAverage(ONE_DAY_MILLIS);
failureRequestsPerSecondLastDay = new MovingAverage(ONE_DAY_MILLIS);
}
private void recordRequest() {
final double[] rates = flowMeter.sample();
totalRequestsPerSecondLastMinute.update(rates[0]);
totalRequestsPerSecondLastHour.update(rates[0]);
totalRequestsPerSecondLastDay.update(rates[0]);
successRequestsPerSecondLastMinute.update(rates[1]);
successRequestsPerSecondLastHour.update(rates[1]);
successRequestsPerSecondLastDay.update(rates[1]);
failureRequestsPerSecondLastMinute.update(rates[2]);
failureRequestsPerSecondLastHour.update(rates[2]);
failureRequestsPerSecondLastDay.update(rates[2]);
}
private void recordSuccess(final LatencyTracker latencyTracker) {
final long successMillis = latencyTracker.getLastSuccessMillis();
averageSuccessLatencyLastMinute.update(successMillis);
averageSuccessLatencyLastHour.update(successMillis);
averageSuccessLatencyLastDay.update(successMillis);
lifetimeSuccessLatencyQuantile.addSample(successMillis);
successLatencyQuantileLastMinute.addSample(successMillis);
successLatencyQuantileLastHour.addSample(successMillis);
successLatencyQuantileLastDay.addSample(successMillis);
lifetimeMaxSuccessMillis = successMillis > lifetimeMaxSuccessMillis
? successMillis : lifetimeMaxSuccessMillis;
recordRequest();
}
private void recordFailure(final LatencyTracker latencyTracker) {
final long failureMillis = latencyTracker.getLastFailureMillis();
averageFailureLatencyLastMinute.update(failureMillis);
averageFailureLatencyLastHour.update(failureMillis);
averageFailureLatencyLastDay.update(failureMillis);
lifetimeFailureLatencyQuantile.addSample(failureMillis);
failureLatencyQuantileLastMinute.addSample(failureMillis);
failureLatencyQuantileLastHour.addSample(failureMillis);
failureLatencyQuantileLastDay.addSample(failureMillis);
lifetimeMaxFailureMillis = failureMillis > lifetimeMaxFailureMillis
? failureMillis : lifetimeMaxFailureMillis;
recordRequest();
}
@Override
public <T> T invoke(final Callable<T> c) throws Exception {
final LatencyTracker latencyTracker = new LatencyTracker();
try {
final T result = requestCounter.invoke(new Callable<T>() {
@Override
public T call() throws Exception {
return latencyTracker.invoke(c);
}
});
recordSuccess(latencyTracker);
return result;
} catch (final Exception e) {
recordFailure(latencyTracker);
if (WRAP_MSG.equals(e.getMessage())) {
throw (Exception) e.getCause();
}
throw e;
}
}
@Override
public void invoke(final Runnable r) throws Exception {
final LatencyTracker latencyTracker = new LatencyTracker();
try {
requestCounter.invoke(new Runnable() {
@Override
public void run() {
try {
latencyTracker.invoke(r);
} catch (final Exception e) {
throw new RuntimeException(WRAP_MSG, e);
}
}
});
recordSuccess(latencyTracker);
} catch (final RuntimeException re) {
recordFailure(latencyTracker);
if (WRAP_MSG.equals(re.getMessage())) {
throw (Exception) re.getCause();
}
throw re;
}
}
@Override
public <T> T invoke(final Runnable r, final T result) throws Exception {
this.invoke(r);
return result;
}
/**
* Returns the average latency in milliseconds of a successful request, as measured
* over the last minute.
*
* @return double
*/
public double getAverageSuccessLatencyLastMinute() {
return averageSuccessLatencyLastMinute.getAverage();
}
/**
* Returns the average latency in milliseconds of a successful request, as measured
* over the last hour.
*
* @return double
*/
public double getAverageSuccessLatencyLastHour() {
return averageSuccessLatencyLastHour.getAverage();
}
/**
* Returns the average latency in milliseconds of a successful request, as measured
* over the last day.
*
* @return double
*/
public double getAverageSuccessLatencyLastDay() {
return averageSuccessLatencyLastDay.getAverage();
}
/**
* Returns the average latency in milliseconds of a failed request, as measured over
* the last minute.
*
* @return double
*/
public double getAverageFailureLatencyLastMinute() {
return averageFailureLatencyLastMinute.getAverage();
}
/**
* Returns the average latency in milliseconds of a failed request, as measured over
* the last hour.
*
* @return double
*/
public double getAverageFailureLatencyLastHour() {
return averageFailureLatencyLastHour.getAverage();
}
/**
* Returns the average latency in milliseconds of a failed request, as measured over
* the last day.
*
* @return double
*/
public double getAverageFailureLatencyLastDay() {
return averageFailureLatencyLastDay.getAverage();
}
/**
* Returns the average request rate in requests per second of all requests, as
* measured over the last minute.
*
* @return double
*/
public double getTotalRequestsPerSecondLastMinute() {
return totalRequestsPerSecondLastMinute.getAverage();
}
/**
* Returns the average request rate in requests per second of successful requests, as
* measured over the last minute.
*
* @return double
*/
public double getSuccessRequestsPerSecondLastMinute() {
return successRequestsPerSecondLastMinute.getAverage();
}
/**
* Returns the average request rate in requests per second of failed requests, as
* measured over the last minute.
*
* @return double
*/
public double getFailureRequestsPerSecondLastMinute() {
return failureRequestsPerSecondLastMinute.getAverage();
}
/**
* Returns the average request rate in requests per second of all requests, as
* measured over the last hour.
*
* @return double
*/
public double getTotalRequestsPerSecondLastHour() {
return totalRequestsPerSecondLastHour.getAverage();
}
/**
* Returns the average request rate in requests per second of successful requests, as
* measured over the last hour.
*
* @return double
*/
public double getSuccessRequestsPerSecondLastHour() {
return successRequestsPerSecondLastHour.getAverage();
}
/**
* Returns the average request rate in requests per second of failed requests, as
* measured over the last hour.
*
* @return double
*/
public double getFailureRequestsPerSecondLastHour() {
return failureRequestsPerSecondLastHour.getAverage();
}
/**
* Returns the average request rate in requests per second of all requests, as
* measured over the last day.
*
* @return double
*/
public double getTotalRequestsPerSecondLastDay() {
return totalRequestsPerSecondLastDay.getAverage();
}
/**
* Returns the average request rate in requests per second of successful requests, as
* measured over the last day.
*
* @return double
*/
public double getSuccessRequestsPerSecondLastDay() {
return successRequestsPerSecondLastDay.getAverage();
}
/**
* Returns the average request rate in requests per second of failed requests, as
* measured over the last day.
*
* @return double
*/
public double getFailureRequestsPerSecondLastDay() {
return failureRequestsPerSecondLastDay.getAverage();
}
/**
* Returns the average request rate in requests per second of all requests, as
* measured since this object was initialized.
*
* @return double
*/
public double getTotalRequestsPerSecondLifetime() {
final long deltaT = System.currentTimeMillis() - startupMillis;
return (double) requestCounter.sample()[0] / (double) deltaT * 1000;
}
/**
* Returns the average request rate in requests per second of successful requests, as
* measured since this object was initialized.
*
* @return double
*/
public double getSuccessRequestsPerSecondLifetime() {
final long deltaT = System.currentTimeMillis() - startupMillis;
return (double) requestCounter.sample()[1] / (double) deltaT * 1000;
}
/**
* Returns the average request rate in requests per second of failed requests, as
* measured since this object was initialized.
*
* @return double
*/
public double getFailureRequestsPerSecondLifetime() {
final long deltaT = System.currentTimeMillis() - startupMillis;
return (double) requestCounter.sample()[2] / (double) deltaT * 1000;
}
/**
* Returns the underlying request counter that this performance monitor is using. This
* can be used in conjunction with {@link PercentErrPerTimeFailureInterpreter}.
*
* @return RequestCounter the request count tracker class
*/
public RequestCounter getRequestCounter() {
return requestCounter;
}
/**
* Returns the total number of requests seen by this {@link PerformanceMonitor}.
*
* @return long
*/
public long getRequestCount() {
return requestCounter.sample()[0];
}
/**
* Returns the number of successful requests seen by this {@link PerformanceMonitor}.
*
* @return long
*/
public long getSuccessCount() {
return requestCounter.sample()[1];
}
/**
* Returns the number of failed requests seen by this {@link PerformanceMonitor}.
*
* @return long
*/
public long getFailureCount() {
return requestCounter.sample()[2];
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for successful
* requests.
*
* @return latency in milliseconds
*/
public long getMedianPercentileSuccessLatencyLifetime() {
return lifetimeSuccessLatencyQuantile.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests.
*
* @return latency in milliseconds
*/
public long get95thPercentileSuccessLatencyLifetime() {
return lifetimeSuccessLatencyQuantile.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests.
*
* @return latency in milliseconds
*/
public long get99thPercentileSuccessLatencyLifetime() {
return lifetimeSuccessLatencyQuantile.getPercentile(99);
}
/**
* Returns the maximum latency seen by this {@link PerformanceMonitor} for successful
* requests.
*
* @return latency in milliseconds
*/
public long getMaxSuccessLatencyLifetime() {
return lifetimeMaxSuccessMillis;
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for successful
* requests over the last minute.
*
* @return latency in milliseconds
*/
public long getMedianPercentileSuccessLatencyLastMinute() {
return successLatencyQuantileLastMinute.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests over the last minute.
*
* @return latency in milliseconds
*/
public long get95thPercentileSuccessLatencyLastMinute() {
return successLatencyQuantileLastMinute.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests over the last minute.
*
* @return latency in milliseconds
*/
public long get99thPercentileSuccessLatencyLastMinute() {
return successLatencyQuantileLastMinute.getPercentile(99);
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for successful
* requests over the last hour.
*
* @return latency in milliseconds
*/
public long getMedianPercentileSuccessfulLatencyLastHour() {
return successLatencyQuantileLastHour.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests over the last hour.
*
* @return latency in milliseconds
*/
public long get95thPercentileSuccessLatencyLastHour() {
return successLatencyQuantileLastHour.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests over the last hour.
*
* @return latency in milliseconds
*/
public long get99thPercentileSuccessLatencyLastHour() {
return successLatencyQuantileLastHour.getPercentile(99);
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for successful
* requests over the last day.
*
* @return latency in milliseconds
*/
public long getMedianPercentileSuccessLatencyLastDay() {
return successLatencyQuantileLastDay.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests over the last day.
*
* @return latency in milliseconds
*/
public long get95thPercentileSuccessLatencyLastDay() {
return successLatencyQuantileLastDay.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* successful requests over the last hour.
*
* @return latency in milliseconds
*/
public long get99thPercentileSuccessLatencyLastDay() {
return successLatencyQuantileLastDay.getPercentile(99);
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for failed
* requests.
*
* @return latency in milliseconds
*/
public long getMedianPercentileFailureLatencyLifetime() {
return lifetimeFailureLatencyQuantile.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests.
*
* @return latency in milliseconds
*/
public long get95thPercentileFailureLatencyLifetime() {
return lifetimeFailureLatencyQuantile.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests.
*
* @return latency in milliseconds
*/
public long get99thPercentileFailureLatencyLifetime() {
return lifetimeFailureLatencyQuantile.getPercentile(99);
}
/**
* Returns the maximum latency seen by this {@link PerformanceMonitor} for failed
* requests.
*
* @return latency in milliseconds
*/
public long getMaxFailureLatencyLifetime() {
return lifetimeMaxFailureMillis;
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for failed
* requests over the last minute.
*
* @return latency in milliseconds
*/
public long getMedianPercentileFailureLatencyLastMinute() {
return failureLatencyQuantileLastMinute.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests over the last minute.
*
* @return latency in milliseconds
*/
public long get95thPercentileFailureLatencyLastMinute() {
return failureLatencyQuantileLastMinute.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests over the last minute.
*
* @return latency in milliseconds
*/
public long get99thPercentileFailureLatencyLastMinute() {
return failureLatencyQuantileLastMinute.getPercentile(99);
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for failed
* requests over the last hour.
*
* @return latency in milliseconds
*/
public long getMedianPercentileFailureLatencyLastHour() {
return failureLatencyQuantileLastHour.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests over the last hour.
*
* @return latency in milliseconds
*/
public long get95thPercentileFailureLatencyLastHour() {
return failureLatencyQuantileLastHour.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests over the last hour.
*
* @return latency in milliseconds
*/
public long get99thPercentileFailureLatencyLastHour() {
return failureLatencyQuantileLastHour.getPercentile(99);
}
/**
* Returns the median latency seen by this {@link PerformanceMonitor} for failed
* requests over the last day.
*
* @return latency in milliseconds
*/
public long getMedianPercentileFailureLatencyLastDay() {
return failureLatencyQuantileLastDay.getPercentile(50);
}
/**
* Returns the 95th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests over the last day.
*
* @return latency in milliseconds
*/
public long get95thPercentileFailureLatencyLastDay() {
return failureLatencyQuantileLastDay.getPercentile(95);
}
/**
* Returns the 99th-percentile latency seen by this {@link PerformanceMonitor} for
* failed requests over the last hour.
*
* @return latency in milliseconds
*/
public long get99thPercentileFailureLatencyLastDay() {
return failureLatencyQuantileLastDay.getPercentile(99);
}
}