package org.apache.cassandra.stress;
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import java.io.PrintStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadFactory;
import org.apache.cassandra.stress.util.*;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.stress.settings.StressSettings;
public class StressMetrics
{
private static final ThreadFactory tf = new NamedThreadFactory("StressMetrics");
private final PrintStream output;
private final Thread thread;
private volatile boolean stop = false;
private volatile boolean cancelled = false;
private final Uncertainty rowRateUncertainty = new Uncertainty();
private final CountDownLatch stopped = new CountDownLatch(1);
private final Timing timing;
private final Callable<JmxCollector.GcStats> gcStatsCollector;
private volatile JmxCollector.GcStats totalGcStats;
private final StressSettings settings;
public StressMetrics(PrintStream output, final long logIntervalMillis, StressSettings settings)
{
this.output = output;
this.settings = settings;
Callable<JmxCollector.GcStats> gcStatsCollector;
totalGcStats = new JmxCollector.GcStats(0);
try
{
gcStatsCollector = new JmxCollector(settings.node.resolveAllPermitted(settings), settings.port.jmxPort);
}
catch (Throwable t)
{
switch (settings.log.level)
{
case VERBOSE:
t.printStackTrace();
}
System.err.println("Failed to connect over JMX; not collecting these stats");
gcStatsCollector = new Callable<JmxCollector.GcStats>()
{
public JmxCollector.GcStats call() throws Exception
{
return totalGcStats;
}
};
}
this.gcStatsCollector = gcStatsCollector;
this.timing = new Timing(settings.samples.historyCount, settings.samples.reportCount);
printHeader("", output);
thread = tf.newThread(new Runnable()
{
@Override
public void run()
{
timing.start();
try {
while (!stop)
{
try
{
long sleepNanos = timing.getHistory().endNanos() - System.nanoTime();
long sleep = (sleepNanos / 1000000) + logIntervalMillis;
if (sleep < logIntervalMillis >>> 3)
// if had a major hiccup, sleep full interval
Thread.sleep(logIntervalMillis);
else
Thread.sleep(sleep);
update();
} catch (InterruptedException e)
{
break;
}
}
update();
}
catch (InterruptedException e)
{}
catch (Exception e)
{
cancel();
e.printStackTrace(StressMetrics.this.output);
}
finally
{
rowRateUncertainty.wakeAll();
stopped.countDown();
}
}
});
}
public void start()
{
thread.start();
}
public void waitUntilConverges(double targetUncertainty, int minMeasurements, int maxMeasurements) throws InterruptedException
{
rowRateUncertainty.await(targetUncertainty, minMeasurements, maxMeasurements);
}
public void cancel()
{
cancelled = true;
stop = true;
thread.interrupt();
rowRateUncertainty.wakeAll();
}
public void stop() throws InterruptedException
{
stop = true;
thread.interrupt();
stopped.await();
}
private void update() throws InterruptedException
{
Timing.TimingResult<JmxCollector.GcStats> result = timing.snap(gcStatsCollector);
totalGcStats = JmxCollector.GcStats.aggregate(Arrays.asList(totalGcStats, result.extra));
TimingInterval current = result.intervals.combine(settings.samples.reportCount);
TimingInterval history = timing.getHistory().combine(settings.samples.historyCount);
rowRateUncertainty.update(current.adjustedRowRate());
if (current.operationCount != 0)
{
if (result.intervals.intervals().size() > 1)
{
for (Map.Entry<String, TimingInterval> type : result.intervals.intervals().entrySet())
printRow("", type.getKey(), type.getValue(), timing.getHistory().get(type.getKey()), result.extra, rowRateUncertainty, output);
}
printRow("", "total", current, history, result.extra, rowRateUncertainty, output);
}
if (timing.done())
stop = true;
}
// PRINT FORMATTING
public static final String HEADFORMAT = "%-10s%10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s,%7s,%7s,%8s,%8s,%8s,%8s";
public static final String ROWFORMAT = "%-10s%10d,%8.0f,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f,%7d,%7.0f,%8.0f,%8.0f,%8.0f,%8.0f";
private static void printHeader(String prefix, PrintStream output)
{
output.println(prefix + String.format(HEADFORMAT, "type,", "total ops","op/s","pk/s","row/s","mean","med",".95",".99",".999","max","time","stderr", "errors", "gc: #", "max ms", "sum ms", "sdv ms", "mb"));
}
private static void printRow(String prefix, String type, TimingInterval interval, TimingInterval total, JmxCollector.GcStats gcStats, Uncertainty opRateUncertainty, PrintStream output)
{
output.println(prefix + String.format(ROWFORMAT,
type + ",",
total.operationCount,
interval.opRate(),
interval.partitionRate(),
interval.rowRate(),
interval.meanLatency(),
interval.medianLatency(),
interval.rankLatency(0.95f),
interval.rankLatency(0.99f),
interval.rankLatency(0.999f),
interval.maxLatency(),
total.runTime() / 1000f,
opRateUncertainty.getUncertainty(),
interval.errorCount,
gcStats.count,
gcStats.maxms,
gcStats.summs,
gcStats.sdvms,
gcStats.bytes / (1 << 20)
));
}
public void summarise()
{
output.println("\n");
output.println("Results:");
TimingIntervals opHistory = timing.getHistory();
TimingInterval history = opHistory.combine(settings.samples.historyCount);
output.println(String.format("op rate : %.0f %s", history.opRate(), opHistory.opRates()));
output.println(String.format("partition rate : %.0f %s", history.partitionRate(), opHistory.partitionRates()));
output.println(String.format("row rate : %.0f %s", history.rowRate(), opHistory.rowRates()));
output.println(String.format("latency mean : %.1f %s", history.meanLatency(), opHistory.meanLatencies()));
output.println(String.format("latency median : %.1f %s", history.medianLatency(), opHistory.medianLatencies()));
output.println(String.format("latency 95th percentile : %.1f %s", history.rankLatency(.95f), opHistory.rankLatencies(0.95f)));
output.println(String.format("latency 99th percentile : %.1f %s", history.rankLatency(0.99f), opHistory.rankLatencies(0.99f)));
output.println(String.format("latency 99.9th percentile : %.1f %s", history.rankLatency(0.999f), opHistory.rankLatencies(0.999f)));
output.println(String.format("latency max : %.1f %s", history.maxLatency(), opHistory.maxLatencies()));
output.println(String.format("Total partitions : %d %s", history.partitionCount, opHistory.partitionCounts()));
output.println(String.format("Total errors : %d %s", history.errorCount, opHistory.errorCounts()));
output.println(String.format("total gc count : %.0f", totalGcStats.count));
output.println(String.format("total gc mb : %.0f", totalGcStats.bytes / (1 << 20)));
output.println(String.format("total gc time (s) : %.0f", totalGcStats.summs / 1000));
output.println(String.format("avg gc time(ms) : %.0f", totalGcStats.summs / totalGcStats.count));
output.println(String.format("stdev gc time(ms) : %.0f", totalGcStats.sdvms));
output.println("Total operation time : " + DurationFormatUtils.formatDuration(
history.runTime(), "HH:mm:ss", true));
}
public static void summarise(List<String> ids, List<StressMetrics> summarise, PrintStream out, int historySampleCount)
{
int idLen = 0;
for (String id : ids)
idLen = Math.max(id.length(), idLen);
String formatstr = "%" + idLen + "s, ";
printHeader(String.format(formatstr, "id"), out);
for (int i = 0 ; i < ids.size() ; i++)
{
for (Map.Entry<String, TimingInterval> type : summarise.get(i).timing.getHistory().intervals().entrySet())
{
printRow(String.format(formatstr, ids.get(i)),
type.getKey(),
type.getValue(),
type.getValue(),
summarise.get(i).totalGcStats,
summarise.get(i).rowRateUncertainty,
out);
}
TimingInterval hist = summarise.get(i).timing.getHistory().combine(historySampleCount);
printRow(String.format(formatstr, ids.get(i)),
"total",
hist,
hist,
summarise.get(i).totalGcStats,
summarise.get(i).rowRateUncertainty,
out
);
}
}
public Timing getTiming()
{
return timing;
}
public boolean wasCancelled()
{
return cancelled;
}
}