/*******************************************************************************
* Copyright (c) 1998, 2015 Oracle and/or its affiliates. All rights reserved.
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v1.0 and Eclipse Distribution License v. 1.0
* which accompanies this distribution.
* The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v10.html
* and the Eclipse Distribution License is available at
* http://www.eclipse.org/org/documents/edl-v10.php.
*
* Contributors:
* Oracle - initial API and implementation from Oracle TopLink
******************************************************************************/
package org.eclipse.persistence.testing.framework;
import org.eclipse.persistence.sessions.*;
import org.eclipse.persistence.expressions.*;
import org.eclipse.persistence.queries.*;
/**
* Used to ensure the performance of an operation or fine grained use case does not
* regress / become slower than the previous successful run of the test.
* This accesses the test result database to query the previous test result.
*/
public abstract class PerformanceRegressionTestCase extends PerformanceComparisonTestCase implements PerformanceRegressionTest {
/** Switch reset to true to reset the baseline comparison. */
public static boolean reset = false;
public PerformanceRegressionTestCase() {
this.allowableDecrease = DEFAULT_ALLOWABLE_DECREASE;
}
/**
* Load the last test result from the test result database.
* Find only the results run on the same machine and database.
* Compare the current test run result with the previous results
* do determine if the test passes or fails.
*/
public void verify() {
super.verify();
verify(this);
}
/**
* Load the last test result from the test result database.
* Find only the results run on the same machine and database.
* Compare the current test run result with the previous results
* do determine if the test passes or fails.
*/
public static void verify(PerformanceRegressionTest test) {
// Ensures all tests pass to reset baseline,
// Required when tests or environment change to be slower to avoid failures.
if (reset) {
throw new TestWarningException("Reseting baseline.");
}
Session session = LoadBuildSystem.getSystem().getSession();
// Query all previous successful test results for this test on the same machine and database.
// Find only the baseline version, or last version run different than current version.
// If you need to compare results against the current version you must change the TopLink version string.
ReadAllQuery query = new ReadAllQuery(TestResult.class);
ExpressionBuilder result = new ExpressionBuilder();
query.setSelectionCriteria(result.get("name").equal(test.getName()).and(result.get("loadBuildSummary").get("machine").equal(LoadBuildSystem.getSummary().getMachine())).and(result.get("loadBuildSummary").get("loginChoice").equal(LoadBuildSystem.getSummary().getLoginChoice())));
// Allow comparing to a set version through a system property.
String currentVersion = LoadBuildSystem.getSummary().getToplinkVersion();
String baselineVersion = null;
if (System.getProperties().containsKey("toplink.loadbuild.baseline-version")) {
baselineVersion = System.getProperties().getProperty("toplink.loadbuild.baseline-version");
// System properties cannot store spaces so need to replace them from \b.
baselineVersion = baselineVersion.replace('_', ' ');
((PerformanceComparisonTestResult)((TestCase)test).getTestResult()).baselineVersion = baselineVersion;
} else {
// Compare against the last successful version.
ReportQuery reportQuery = new ReportQuery(TestResult.class, query.getExpressionBuilder());
reportQuery.useDistinct();
reportQuery.returnSingleValue();
reportQuery.addAttribute("version", result.get("loadBuildSummary").get("toplinkVersion"));
reportQuery.setSelectionCriteria(query.getSelectionCriteria().and((result.get("outcome").equal(TestResult.PASSED)).or(result.get("outcome").equal(TestResult.WARNING))).and(result.get("loadBuildSummary").get("toplinkVersion").notEqual(currentVersion)));
reportQuery.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
baselineVersion = (String) session.executeQuery(reportQuery);
}
query.setSelectionCriteria(query.getSelectionCriteria().and(result.get("loadBuildSummary").get("toplinkVersion").equal(baselineVersion)));
query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
query.setMaxRows(10);
query.useCursoredStream(1, 1);
CursoredStream stream = (CursoredStream)session.executeQuery(query);
if (!stream.hasMoreElements()) {
throw new TestWarningException("No previous test result to compare performance with.");
}
TestResult lastResult = (TestResult)stream.nextElement();
double lastCount = lastResult.getTestTime();
PerformanceComparisonTestResult testResult = (PerformanceComparisonTestResult)((TestCase)test).getTestResult();
testResult.getBaselineVersionResults().add(new Double(lastCount));
// Average last 5 runs.
int numberOfRuns = 0;
while (stream.hasMoreElements() && (numberOfRuns < 4)) {
TestResult nextResult = (TestResult)stream.nextElement();
testResult.getBaselineVersionResults().add(new Double(nextResult.getTestTime()));
numberOfRuns++;
}
stream.close();
double baselineAverage = PerformanceComparisonTestResult.averageResults(testResult.getBaselineVersionResults());
double currentCount = ((TestCase)test).getTestResult().getTestTime();
testResult.baselineVersion = lastResult.getLoadBuildSummary().getToplinkVersion();
testResult.percentageDifferenceLastRun =
PerformanceComparisonTestResult.percentageDifference(currentCount, lastCount);
// Query the current version last 5 runs for averaging.
query = new ReadAllQuery(TestResult.class);
result = new ExpressionBuilder();
query.setSelectionCriteria(result.get("name").equal(((TestCase)test).getName()).and(result.get("loadBuildSummary").get("machine").equal(LoadBuildSystem.getSummary().getMachine())).and(result.get("loadBuildSummary").get("loginChoice").equal(LoadBuildSystem.getSummary().getLoginChoice())).and(result.get("loadBuildSummary").get("toplinkVersion").equal(currentVersion)));
query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
query.useCursoredStream(1, 1);
stream = (CursoredStream)session.executeQuery(query);
// Average last 5 runs.
testResult.getCurrentVersionResults().add(new Double(currentCount));
numberOfRuns = 0;
while (stream.hasMoreElements() && (numberOfRuns < 4)) {
TestResult nextResult = (TestResult)stream.nextElement();
testResult.getCurrentVersionResults().add(new Double(nextResult.getTestTime()));
numberOfRuns++;
}
stream.close();
double currentAverage = PerformanceComparisonTestResult.averageResults(testResult.getCurrentVersionResults());
testResult.percentageDifferenceAverage =
PerformanceComparisonTestResult.percentageDifference(currentAverage, baselineAverage);
testResult.baselineStandardDeviation =
PerformanceComparisonTestResult.standardDeviationResults(testResult.getBaselineVersionResults());
testResult.currentStandardDeviation =
PerformanceComparisonTestResult.standardDeviationResults(testResult.getCurrentVersionResults());
if (testResult.percentageDifferenceAverage < test.getAllowableDecrease()) {
throw new TestErrorException("Test is " + ((long)testResult.percentageDifferenceAverage) +
"% slower than last successful execution.");
}
}
}