/*
* The MIT License
*
* Copyright (c) 2010, Yahoo!, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package hudson.plugins.labeledandgroupedtests;
import org.jvnet.hudson.test.HudsonTestCase;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.DomNode;
import com.gargoylesoftware.htmlunit.xml.XmlPage;
import hudson.model.*;
import org.jvnet.hudson.test.recipes.LocalData;
import java.util.concurrent.TimeUnit;
/**
* Test to check our pass/fail/skip counts and grouping
*/
public class CountingTest extends EnhancedHudsonTestCase {
final String TEST_REPORT = "testReport/";
final String API_XML = "api/xml";
final String UNIT_GROUP = "unit/";
final String REGRESSION_GROUP = "regression/";
final String REGGRESION_JUNIT_GROUP = "junit/";
final String UNIT_CPP_GROUP = "cppunit/";
@LocalData
public void SKIP_testOverallCount() throws Exception {
String log = "(no log collected)";
try {
// Load in a project with known counts
FreeStyleProject project = setupProject("knowncount");
assertNotNull(project);
// Run and validate a traditional build
FreeStyleBuild build = project.scheduleBuild2(0).get(5, TimeUnit.MINUTES);
log = getLog(build);
assertBuildStatus(Result.UNSTABLE, build);
HudsonTestCase.WebClient wc = new HudsonTestCase.WebClient();
// Test overall results via xml api
// http://localhost:8080/job/knowncount/5/testReport/api/xml is correct: 69/1/0
XmlPage testReportPage = wc.goToXml(project.getLastBuild().getUrl() + TEST_REPORT + API_XML);
assertXPathValue(testReportPage, "/metaLabeledTestResultGroup/failCount/text()", "1" );
assertXPathValue(testReportPage, "/metaLabeledTestResultGroup/passCount/text()", "69" );
assertXPathValue(testReportPage, "/metaLabeledTestResultGroup/skipCount/text()", "0" );
// Test single-group junit parser results via xml api:
// http://localhost:8080/job/knowncount/5/testReport/regression/regression-hudson.tasks.junit.JUnitParser/api/xml should have 0/1/0
XmlPage regressionJunitPage = wc.goToXml(project.getLastBuild().getUrl() + TEST_REPORT + REGRESSION_GROUP + REGGRESION_JUNIT_GROUP + API_XML);
assertXPathValue(regressionJunitPage, "/testResult/failCount/text()", "1" );
assertXPathValue(regressionJunitPage, "/testResult/passCount/text()", "0" );
assertXPathValue(regressionJunitPage, "/testResult/skipCount/text()", "0" );
// To demonstrate labeled test groups publisher: via xml api, pass count is wrong for a group (but html is ok)
// http://localhost:8080/job/knowncount/5/testReport/unit/api/xml should have 25/0/0, actually has 2/0/0
XmlPage unitReportPage = wc.goToXml(project.getLastBuild().getUrl() + TEST_REPORT + UNIT_GROUP + API_XML);
assertXPathValue(unitReportPage, "/labeledTestResultGroup/failCount/text()", "0" );
assertXPathValue(unitReportPage, "/labeledTestResultGroup/passCount/text()", "25" );
assertXPathValue(unitReportPage, "/labeledTestResultGroup/skipCount/text()", "0" );
// The same problem is visible at these urls, too:
// http://localhost:8080/job/knowncount/5/testReport/regression/api/xml should have 18/1/0, actually has 0/1/0
// http://localhost:8080/job/knowncount/5/testReport/smoke/api/xml should have 26/0/0, actually has 4/0/0
// http://localhost:8080/job/knowncount/5/testReport/unit/unit-hudson.plugins.cppunitparser.CPPUnitTestResultParser/api/xml serves up
XmlPage cppUnitPage = wc.goToXml(project.getLastBuild().getUrl() + TEST_REPORT + UNIT_GROUP + UNIT_CPP_GROUP + API_XML);
assertXPathValue(cppUnitPage, "/CPPUnitTestResultGroup/failCount/text()", "0" );
assertXPathValue(cppUnitPage, "/CPPUnitTestResultGroup/passCount/text()", "23" );
assertXPathValue(cppUnitPage, "/CPPUnitTestResultGroup/skipCount/text()", "0" );
} finally {
System.out.println("Here's the log: " + log);
}
}
/**
* Working on a test for a bug where labeled test groups publisher: inter-build diffs for groups are wrong
* @throws Exception
*/
@LocalData
public void SKIP_testDiffs() throws Exception {
String log = "(no log collected)";
try {
// Load in a project with known counts
FreeStyleProject project = setupProject("knowncount");
assertNotNull(project);
int numBuildsToRun = 4;
// Run and validate a traditional build
FreeStyleBuild build = null;
for (int i = 0; i < numBuildsToRun; i++) {
build = project.scheduleBuild2(0).get(20, TimeUnit.SECONDS);
log = getLog(build);
assertBuildStatus(Result.UNSTABLE, build);
}
HudsonTestCase.WebClient wc = new HudsonTestCase.WebClient();
// First check that for the ones we just ran, it always says (1 failure / �0) on the build summary page
HtmlPage projectPage = wc.goTo(project.getUrl());
String pageAsText = projectPage.asXml();
// System.out.println(pageAsText);
assertXPathResultsContainText( projectPage, "//td[@id='main-panel']", "Test Result");
assertXPathValueContains( projectPage, "//td[@id='main-panel']", "(1 failure ") ;
assertXPathValueContains( projectPage, "//td[@id='main-panel']", "0)");
// Ugh. The +/- character behaves differently on the mac than on unix.
// This test passes on the mac, fails on unix, so, sadly, I'm going to
// do some obnoxious workaround.
String plusOrMinusSymbol = " �";
String xPathToMainPanel = "//td[@id='main-panel']";
new CountingTest.ExpectedValues(projectPage, "Test Result", xPathToMainPanel).runCheck();
new CountingTest.ExpectedValues(projectPage, "(1 failure", xPathToMainPanel).runCheck();
new CountingTest.ExpectedValues(projectPage, "0)", xPathToMainPanel).runCheck();
// TODO: write test: Diffs are wrong for labeled test groups:
// on http://localhost:8080/job/knowncount/6/testReport/ each of unit, regression, smoke
// list +/- as n (+/- n) when in fact it should be n +/- 0
} finally {
System.out.println("Here's the log: " + log);
}
}
class ExpectedValues {
private final String xpath;
private DomNode page;
private final String needle;
private final String msg;
public ExpectedValues( String msg, DomNode page, String needle, String xpath) {
this.needle = needle;
this.page = page;
this.xpath = xpath;
this.msg = msg;
}
public ExpectedValues(DomNode page, String needle, String xpath) {
this("(no message)", page, needle, xpath);
}
public void runCheck() {
assertXPathValueContains(page, xpath, needle);
}
}
}