/*******************************************************************************
* Copyright (c) 1998, 2015 Oracle and/or its affiliates. All rights reserved.
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v1.0 and Eclipse Distribution License v. 1.0
* which accompanies this distribution.
* The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v10.html
* and the Eclipse Distribution License is available at
* http://www.eclipse.org/org/documents/edl-v10.php.
*
* Contributors:
* Oracle - initial API and implementation from Oracle TopLink
******************************************************************************/
package org.eclipse.persistence.testing.framework;
import java.io.*;
import java.util.*;
import org.eclipse.persistence.internal.helper.Helper;
/**
* Stores times for runs of tests vs base-line and computes averges/stats on these.
* The baseline is the first value in the lists, followed by each test's value.
*/
public class PerformanceComparisonTestResult extends TestResult {
public List testCounts;
public List testAverages;
public List testMaxs;
public List testMins;
public List testStandardDeviations;
/**
* Percentage differences between the performance comparisons.
*/
public List percentageDifferences;
/** Records the average test count. */
public double averageTestCount;
/**
* The % difference between the current test average time and the
* last successful run of the test on the baseline version.
* Used for regression tests.
*/
protected double percentageDifferenceLastRun;
/**
* The % difference between the current test average time and the
* average of the last 3 successful runs of the test on the baseline version.
* Used for regression tests.
*/
protected double percentageDifferenceAverage;
/**
* The TopLink version that this test is comparing against.
* This version defaults to the last test run, but can be set through the
* System property toplink.loadbuild.baseline-version to another version such
* as the last release.
* Used for regression tests.
*/
protected String baselineVersion;
/**
* The up to 3 results for the current version.
*/
public List currentVersionResults;
/**
* The up to 3 results for the baseline version compared to.
*/
public List baselineVersionResults;
/**
* The regression baseline % standard deviation.
*/
public double baselineStandardDeviation;
/**
* The regression current % standard deviation.
*/
public double currentStandardDeviation;
public PerformanceComparisonTestResult() {
this.testCounts = new ArrayList();
this.testAverages = new ArrayList();
this.testMaxs = new ArrayList();
this.testMins = new ArrayList();
this.testStandardDeviations = new ArrayList();
this.percentageDifferences = new ArrayList();
this.currentVersionResults = new ArrayList();
this.baselineVersionResults = new ArrayList();
}
public PerformanceComparisonTestResult(TestCase testCase) {
super(testCase);
this.testCounts = new ArrayList();
this.testAverages = new ArrayList();
this.testMaxs = new ArrayList();
this.testMins = new ArrayList();
this.testStandardDeviations = new ArrayList();
this.percentageDifferences = new ArrayList();
this.currentVersionResults = new ArrayList();
this.baselineVersionResults = new ArrayList();
}
public PerformanceComparisonTestResult(TestCase testCase, String result) {
super(testCase, result);
this.testCounts = new ArrayList();
this.testAverages = new ArrayList();
this.testMaxs = new ArrayList();
this.testMins = new ArrayList();
this.testStandardDeviations = new ArrayList();
this.percentageDifferences = new ArrayList();
this.currentVersionResults = new ArrayList();
this.baselineVersionResults = new ArrayList();
}
/**
* Return the average test count.
*/
public double getAverageTestCount() {
return averageTestCount;
}
/**
* Set the average test count.
*/
public void setAverageTestCount(double averageTestCount) {
this.averageTestCount = averageTestCount;
// Also set this as the test time as this is what is stored in the db.
this.testTime = (long)averageTestCount;
}
/**
* Return the % difference between the current test average time and the
* last successful run of the test on the baseline version.
*/
public double getPercentageDifferenceLastRun() {
return percentageDifferenceLastRun;
}
/**
* Return the & difference between the current test average time and the
* average of the last 3 successful runs of the test on the baseline version.
*/
public double getPercentageDifferenceAverage() {
return percentageDifferenceAverage;
}
/**
* Return the TopLink version that this test is comparing against.
* This version defaults to the last test run, but can be set through the
* System property toplink.loadbuild.baseline-version to another version such
* as the last release.
*/
public String getBaselineVersion() {
return baselineVersion;
}
/**
* Compute the test result averages and stand dev.
*/
public void computeResults() {
for (int testIndex = 0; testIndex < getTestCounts().size(); testIndex++) {
List times = (List)getTestCounts().get(testIndex);
double testAverage = PerformanceComparisonTestResult.averageResults(times);
double testStandardDeviation = PerformanceComparisonTestResult.standardDeviationResults(times);
// Set the test average count as the test time.
this.setAverageTestCount((long)testAverage);
this.testAverages.add(new Double(testAverage));
this.testMins.add(PerformanceComparisonTestResult.minResults(times));
this.testMaxs.add(PerformanceComparisonTestResult.maxResults(times));
this.testStandardDeviations.add(new Double(testStandardDeviation));
if (testIndex > 0) {
double testBaseLineAverage = ((Number)this.testAverages.get(0)).doubleValue();
// Difference
double percentageDifference =
PerformanceComparisonTestResult.percentageDifference(testAverage, testBaseLineAverage);
this.percentageDifferences.add(new Double(percentageDifference));
}
}
}
/**
* Return the list of lists of test counts.
*/
public List getTestCounts() {
return testCounts;
}
/**
* Return the regression results from the current version run.
*/
public List getCurrentVersionResults() {
return currentVersionResults;
}
/**
* Return the regression results from the baseline runs.
*/
public List getBaselineVersionResults() {
return baselineVersionResults;
}
/**
* Add the test count for the test index.
*/
public void addTestCount(long time, int test) {
if (getTestCounts().size() <= test) {
getTestCounts().add(new ArrayList());
}
((List)getTestCounts().get(test)).add(new Long(time));
}
/**
* Always log the result for analysis.
*/
public boolean shouldLogResult() {
return true;
}
/**
* logs the result of the test on the print stream.
*/
public void logResult(Writer log) {
String indentationString = getTestCase().getIndentationString();
try {
if (hasError() || hasFatalError() || hasProblem()) {
log.write(indentationString + "##FAILURE##" + Helper.cr());
}
if (!getTestCounts().isEmpty()) {
log.write(indentationString + "RUNS: " + ((List)getTestCounts().get(0)).size() + Helper.cr());
}
for (int index = 0; index < getTestCounts().size(); index++) {
PerformanceComparisonTest test = (PerformanceComparisonTest)this.testCase;
log.write(Helper.cr());
if (index == 0) {
log.write(indentationString + "BASE_LINE TEST" + Helper.cr());
} else {
test = (PerformanceComparisonTest)test.getTests().get(index - 1);
}
log.write(indentationString + "TEST: " + test.getName() + Helper.cr());
log.write(indentationString + "TEST RUN TIME: " + (test.getTestRunTime() / 1000) + " seconds" +
Helper.cr());
log.write(indentationString + "MEAN TEST COUNT: " + this.testAverages.get(index) + Helper.cr());
log.write(indentationString + "MAX TEST COUNT: " + this.testMaxs.get(index) + Helper.cr());
log.write(indentationString + "MIN TEST COUNT: " + this.testMins.get(index) + Helper.cr());
log.write(indentationString + "TEST % STANDARD DEVIATION: " + this.testStandardDeviations.get(index) +
Helper.cr());
if (index > 0) {
log.write(indentationString + "% DIFFERENCE: " + this.percentageDifferences.get(index - 1) +
Helper.cr());
log.write(indentationString + "% DIFFERENCE ALLOWABLE: " + test.getAllowableDecrease() + Helper.cr());
}
}
if (getTestCase() instanceof PerformanceRegressionTest) {
log.write(indentationString + "BASELINE VERSION: " + getBaselineVersion() + Helper.cr());
log.write(indentationString + "BASELINE VERSION RESULTS: " + getBaselineVersionResults() + Helper.cr());
log.write(indentationString + "BASELINE VERSION % STANDARD DEVIATION: " + this.baselineStandardDeviation +
Helper.cr());
log.write(indentationString + "CURRENT VERSION RESULTS: " + this.getCurrentVersionResults() + Helper.cr());
log.write(indentationString + "CURRENT VERSION % STANDARD DEVIATION: " + this.currentStandardDeviation +
Helper.cr());
log.write(indentationString + "% DIFFERENCE (last run): " + getPercentageDifferenceLastRun() + Helper.cr());
log.write(indentationString + "% DIFFERENCE (average): " + getPercentageDifferenceAverage() + Helper.cr());
}
log.write(Helper.cr() + indentationString + "RESULT: " + getOutcome() + Helper.cr());
} catch (IOException exception) {
// Ignore.
} catch (ArrayIndexOutOfBoundsException exception) {
// Ignore.
}
try {
if (getException() != null) {
getException().setIndentationString(indentationString);
log.write(getException() + org.eclipse.persistence.internal.helper.Helper.cr());
}
log.flush();
} catch (IOException exception) {
}
LoadBuildSystem.loadBuild.addResult(this);
}
/**
* Compute the max of the results.
*/
public static Number maxResults(List times) {
Number testMax = new Double(0);
for (int index = 0; index < times.size(); index++) {
Number time = (Number)times.get(index);
if (time.doubleValue() > testMax.doubleValue()) {
testMax = time;
}
}
return testMax;
}
/**
* Compute the min of the results.
*/
public static Number minResults(List times) {
Number testMin = new Double(0);
for (int index = 0; index < times.size(); index++) {
Number time = (Number)times.get(index);
if ((testMin.doubleValue() == 0) || (time.doubleValue() < testMin.doubleValue())) {
testMin = time;
}
}
return testMin;
}
/**
* Filter max and min from results.
*/
public static List filterMaxMinResults(List times) {
List filteredTimes = new ArrayList(times);
if (filteredTimes.size() > 3) {
filteredTimes.remove(maxResults(times));
filteredTimes.remove(minResults(times));
}
return filteredTimes;
}
/**
* Compute the average of the results rejecting the min and max.
*/
public static double averageResults(List allTimes) {
// Compute the average reject the min and max to improve consistency.
List times = filterMaxMinResults(allTimes);
double testAverage = 0;
for (int index = 0; index < times.size(); index++) {
double time = ((Number)times.get(index)).doubleValue();
testAverage = testAverage + time;
}
testAverage = testAverage / times.size();
return testAverage;
}
/**
* Compute the standard deviation of the results rejecting the min and max.
*/
public static double standardDeviationResults(List allTimes) {
// Compute the average reject the min and max to improve consistency.
double testAverage = averageResults(allTimes);
// Compute the standard deviation reject the min and max to improve consistency.
List times = filterMaxMinResults(allTimes);
double testStandardDeviation = 0;
for (int index = 0; index < times.size(); index++) {
double time = ((Number)times.get(index)).doubleValue();
testStandardDeviation = testStandardDeviation + Math.pow(time - testAverage, 2);
}
testStandardDeviation = testStandardDeviation / times.size();
testStandardDeviation = Math.sqrt(testStandardDeviation);
// As percent of average
testStandardDeviation = (testStandardDeviation / testAverage) * 100;
return testStandardDeviation;
}
/**
* Compute the percentage difference between the values.
*/
public static double percentageDifference(double result, double baseline) {
// Difference
double percentageDifference = ((result - baseline) / baseline) * 100;
// If negative calculate from the inverse spective to get a more meaningful result.
if (percentageDifference < 0) {
percentageDifference = -1 * ((baseline - result) / result) * 100;
}
return percentageDifference;
}
}