/* Copyright 2014 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.jenkins.flakyTestHandler.junit;
import com.google.jenkins.flakyTestHandler.junit.FlakyCaseResult.FlakyRunInformation;
import junit.framework.TestCase;
import org.jvnet.hudson.test.Bug;
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.net.URISyntaxException;
import java.util.List;
import edu.umd.cs.findbugs.annotations.SuppressWarnings;
import hudson.XmlFile;
/**
* Test cases for parsing JUnit report XML files. As there are no XML schema for JUnit xml files,
* Hudson needs to handle varied xml files.
*
* https://github.com/jenkinsci/jenkins/blob/master/core/src/test/java/hudson/tasks/junit/
* SuiteResultTest.java
*/
public class FlakySuiteResultTest extends TestCase {
private File getDataFile(String name) throws URISyntaxException {
return new File(FlakySuiteResultTest.class.getResource(name).toURI());
}
private FlakySuiteResult parseOne(File file) throws Exception {
List<FlakySuiteResult> results = FlakySuiteResult.parse(file, false);
assertEquals(1, results.size());
return results.get(0);
}
private List<FlakySuiteResult> parseSuites(File file) throws Exception {
return FlakySuiteResult.parse(file, false);
}
@Bug(1233)
public void testIssue1233() throws Exception {
FlakySuiteResult result = parseOne(getDataFile("junit-report-1233.xml"));
List<FlakyCaseResult> cases = result.getCases();
assertEquals("Class name is incorrect", "test.foo.bar.DefaultIntegrationTest",
cases.get(0).getClassName());
assertEquals("Class name is incorrect", "test.foo.bar.BundleResolverIntegrationTest",
cases.get(1).getClassName());
assertEquals("Class name is incorrect", "test.foo.bar.BundleResolverIntegrationTest",
cases.get(2).getClassName());
assertEquals("Class name is incorrect", "test.foo.bar.ProjectSettingsTest",
cases.get(3).getClassName());
assertEquals("Class name is incorrect", "test.foo.bar.ProjectSettingsTest",
cases.get(4).getClassName());
}
/**
* JUnit report file is generated by SoapUI Pro 1.7.6
*/
@Bug(1463)
public void testIssue1463() throws Exception {
FlakySuiteResult result = parseOne(getDataFile("junit-report-1463.xml"));
List<FlakyCaseResult> cases = result.getCases();
for (FlakyCaseResult caseResult : cases) {
assertEquals("Test class name is incorrect in " + caseResult.getName(), "WLI-FI-Tests-Fake",
caseResult.getClassName());
}
assertEquals("Test name is incorrect", "IF_importTradeConfirmationToDwh",
cases.get(0).getName());
assertEquals("Test name is incorrect", "IF_getAmartaDisbursements", cases.get(1).getName());
assertEquals("Test name is incorrect", "IF_importGLReconDataToDwh", cases.get(2).getName());
assertEquals("Test name is incorrect", "IF_importTradeInstructionsToDwh",
cases.get(3).getName());
assertEquals("Test name is incorrect", "IF_getDeviationTradeInstructions",
cases.get(4).getName());
assertEquals("Test name is incorrect", "IF_getDwhGLData", cases.get(5).getName());
}
/**
* JUnit report produced by TAP (Test Anything Protocol)
*/
@Bug(1472)
public void testIssue1472() throws Exception {
List<FlakySuiteResult> results = FlakySuiteResult
.parse(getDataFile("junit-report-1472.xml"), false);
assertTrue(results.size() > 20); // lots of data here
FlakySuiteResult sr0 = results.get(0);
FlakySuiteResult sr1 = results.get(1);
assertEquals("make_test.t_basic_lint_t", sr0.getName());
assertEquals("make_test.t_basic_meta_t", sr1.getName());
assertTrue(!sr0.getStdout().equals(sr1.getStdout()));
}
@Bug(2874)
public void testIssue2874() throws Exception {
FlakySuiteResult result = parseOne(getDataFile("junit-report-2874.xml"));
assertEquals("test suite name", "DummyTest", result.getName());
}
public void testErrorDetails() throws Exception {
FlakySuiteResult result = parseOne(getDataFile("junit-report-errror-details.xml"));
List<FlakyCaseResult> cases = result.getCases();
for (FlakyCaseResult caseResult : cases) {
assertEquals("Test class name is incorrect in " + caseResult.getName(),
"some.package.somewhere.WhooHoo", caseResult.getClassName());
}
assertEquals("this normally has the string like, expected mullet, but got bream",
cases.get(0).getErrorDetails());
}
public void testFlakySuiteResultPersistence() throws Exception {
FlakySuiteResult source = parseOne(getDataFile("junit-report-1233.xml"));
File dest = File.createTempFile("testFlakySuiteResultPersistence", ".xml");
try {
XmlFile xmlFile = new XmlFile(dest);
xmlFile.write(source);
FlakySuiteResult result = (FlakySuiteResult) xmlFile.read();
assertNotNull(result);
assertEquals(source.getName(), result.getName());
assertEquals(source.getTimestamp(), result.getTimestamp());
assertEquals(source.getDuration(), result.getDuration());
assertEquals(source.getStderr(), result.getStderr());
assertEquals(source.getStdout(), result.getStdout());
assertEquals(source.getCases().size(), result.getCases().size());
assertNotNull(result.getCase("testGetBundle"));
} finally {
dest.delete();
}
}
//@Bug(6516)
public void testSuiteStdioTrimming() throws Exception {
File data = File.createTempFile("testSuiteStdioTrimming", ".xml");
try {
Writer w = new FileWriter(data);
try {
PrintWriter pw = new PrintWriter(w);
pw.println("<testsuites name='x'>");
pw.println("<testsuite failures='0' errors='0' tests='1' name='x'>");
pw.println("<testcase name='x' classname='x'/>");
pw.println("<system-out/>");
pw.print("<system-err><![CDATA[");
pw.println("First line is intact.");
for (int i = 0; i < 100; i++) {
pw.println("Line #" + i + " might be elided.");
}
pw.println("Last line is intact.");
pw.println("]]></system-err>");
pw.println("</testsuite>");
pw.println("</testsuites>");
pw.flush();
} finally {
w.close();
}
FlakySuiteResult sr = parseOne(data);
assertEquals(sr.getStderr(), 1030, sr.getStderr().length());
} finally {
data.delete();
}
}
@SuppressWarnings(
{"RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", "DM_DEFAULT_ENCODING", "OS_OPEN_STREAM"})
public void testSuiteStdioTrimmingSurefire() throws Exception {
File data = File.createTempFile("TEST-", ".xml");
try {
Writer w = new FileWriter(data);
try {
PrintWriter pw = new PrintWriter(w);
pw.println("<testsuites name='x'>");
pw.println("<testsuite failures='0' errors='0' tests='1' name='x'>");
pw.println("<testcase name='x' classname='x'/>");
pw.println("</testsuite>");
pw.println("</testsuites>");
pw.flush();
} finally {
w.close();
}
File data2 = new File(data.getParentFile(),
data.getName().replaceFirst("^TEST-(.+)[.]xml$", "$1-output.txt"));
try {
w = new FileWriter(data2);
try {
PrintWriter pw = new PrintWriter(w);
pw.println("First line is intact.");
for (int i = 0; i < 100; i++) {
pw.println("Line #" + i + " might be elided.");
}
pw.println("Last line is intact.");
pw.flush();
} finally {
w.close();
}
FlakySuiteResult sr = parseOne(data);
assertEquals(sr.getStdout(), 1030, sr.getStdout().length());
} finally {
data2.delete();
}
} finally {
data.delete();
}
}
/**
* When the testcase fails to initialize (exception in constructor or @Before) there is no
* 'testcase' element at all.
*/
@Bug(6700)
public void testErrorInTestInitialization() throws Exception {
FlakySuiteResult FlakySuiteResult = parseOne(getDataFile("junit-report-6700.xml"));
assertEquals(1, FlakySuiteResult.getCases().size());
FlakyCaseResult result = FlakySuiteResult.getCases().get(0);
assertEquals(1, result.getFailCount());
assertTrue(result.getErrorStackTrace() != null);
}
@Bug(6454)
public void testParseNestedTestSuites() throws Exception {
// A report with several nested suites
// 3 of them have actual some tests - each exactly one
List<FlakySuiteResult> results = parseSuites(getDataFile("junit-report-nested-testsuites.xml"));
assertEquals(3, results.size());
for (FlakySuiteResult result : results) {
assertEquals(1, result.getCases().size());
}
}
/**
* Test parsing of flaky test report
*/
public void testParsingFlakyReports() throws Exception {
FlakySuiteResult result = parseOne(getDataFile("flaky-reports/flaky-report-1.xml"));
List<FlakyCaseResult> cases = result.getCases();
assertEquals("Class name is incorrect", "test.foo.bar.DefaultIntegrationTest",
cases.get(0).getClassName());
assertTrue("Flaky test is not parsed correctly", cases.get(0).isFlaked());
assertEquals("Incorrect number of flaky run information", 2,
cases.get(0).getFlakyRuns().size());
// Parsing flakyFailure
FlakyRunInformation runInformation = cases.get(0).getFlakyRuns().get(0);
assertEquals("Error message of flaky rerun is incorrect", "flaky failure 1",
runInformation.getFlakyErrorDetails());
assertEquals("Stack trace of flaky rerun is incorrect", "flaky stacktrace",
runInformation.getFlakyErrorStackTrace().trim());
assertEquals("System-err of flaky rerun is incorrect", "flaky system err",
runInformation.getFlakyStdErr());
assertEquals("System-out of flaky rerun is incorrect", "flaky system out",
runInformation.getFlakyStdOut());
// Parsing flakyError
runInformation = cases.get(0).getFlakyRuns().get(1);
assertEquals("Error message of flaky rerun is incorrect", "flaky error 1",
runInformation.getFlakyErrorDetails());
assertEquals("Stack trace of flaky rerun is incorrect", "error stacktrace",
runInformation.getFlakyErrorStackTrace().trim());
assertEquals("System-err of flaky rerun is incorrect", "error system err",
runInformation.getFlakyStdErr());
assertEquals("System-out of flaky rerun is incorrect", "error system out",
runInformation.getFlakyStdOut());
assertEquals("Class name is incorrect", "test.foo.bar.BundleResolverIntegrationTest",
cases.get(1).getClassName());
assertFalse("Flaky test is not parsed correctly", cases.get(1).isFlaked());
assertTrue("Flaky test is not parsed correctly", cases.get(1).isFailed());
assertEquals("Incorrect number of flaky run information", 2,
cases.get(1).getFlakyRuns().size());
// Parsing error messages from the original run
assertEquals("Error message of failing tests with rerun is incorrect", "failure",
cases.get(1).getErrorDetails());
assertEquals("Stack trace of failing tests with rerun is incorrect", "stacktrace",
cases.get(1).getErrorStackTrace().trim());
assertEquals("System-err of failing tests with rerun is incorrect", "failure system err",
cases.get(1).getStderr());
assertEquals("System-out of failing tests with rerun is incorrect", "failure system out",
cases.get(1).getStdout());
// Parsing rerunFailure
runInformation = cases.get(1).getFlakyRuns().get(0);
assertEquals("Error message of flaky rerun is incorrect", "flaky failure 2",
runInformation.getFlakyErrorDetails());
assertEquals("Stack trace of flaky rerun is incorrect", "flaky stacktrace 2",
runInformation.getFlakyErrorStackTrace().trim());
assertEquals("System-err of flaky rerun is incorrect", "flaky system err 2",
runInformation.getFlakyStdErr());
assertEquals("System-out of flaky rerun is incorrect", "flaky system out 2",
runInformation.getFlakyStdOut());
// Parsing rerunError
runInformation = cases.get(1).getFlakyRuns().get(1);
assertEquals("Error message of flaky rerun is incorrect", "flaky error 2",
runInformation.getFlakyErrorDetails());
assertEquals("Stack trace of flaky rerun is incorrect", "error stacktrace 2",
runInformation.getFlakyErrorStackTrace().trim());
assertEquals("System-err of flaky rerun is incorrect", "error system err 2",
runInformation.getFlakyStdErr());
assertEquals("System-out of flaky rerun is incorrect", "error system out 2",
runInformation.getFlakyStdOut());
assertFalse("Flaky test is not parsed correctly", cases.get(1).isFlaked());
assertEquals("Class name is incorrect", "test.foo.bar.BundleResolverIntegrationTest",
cases.get(2).getClassName());
assertEquals("Class name is incorrect", "test.foo.bar.ProjectSettingsTest",
cases.get(3).getClassName());
assertEquals("Class name is incorrect", "test.foo.bar.ProjectSettingsTest",
cases.get(4).getClassName());
}
}