/*
* Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package jdk.nashorn.internal.test.framework;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK;
import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringReader;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jdk.nashorn.internal.test.framework.TestFinder.TestFactory;
/**
* Parallel test runner runs tests in multiple threads - but avoids any dependency
* on third-party test framework library such as TestNG.
*/
@SuppressWarnings("javadoc")
public class ParallelTestRunner {
// ParallelTestRunner-specific
private static final String TEST_JS_THREADS = "test.js.threads";
private static final String TEST_JS_REPORT_FILE = "test.js.report.file";
// test262 does a lot of eval's and the JVM hates multithreaded class definition, so lower thread count is usually faster.
private static final int THREADS = Integer.getInteger(TEST_JS_THREADS, Runtime.getRuntime().availableProcessors() > 4 ? 4 : 2);
private final List<ScriptRunnable> tests = new ArrayList<>();
private final Set<String> orphans = new TreeSet<>();
private final ExecutorService executor = Executors.newFixedThreadPool(THREADS);
// Ctrl-C handling
private final CountDownLatch finishedLatch = new CountDownLatch(1);
private final Thread shutdownHook = new Thread() {
@Override
public void run() {
if (!executor.isTerminated()) {
executor.shutdownNow();
try {
executor.awaitTermination(25, TimeUnit.SECONDS);
finishedLatch.await(5, TimeUnit.SECONDS);
} catch (final InterruptedException e) {
// empty
}
}
}
};
public ParallelTestRunner() throws Exception {
suite();
}
private static PrintStream outputStream() {
final String reportFile = System.getProperty(TEST_JS_REPORT_FILE, "");
PrintStream output = System.out;
if (!reportFile.isEmpty()) {
try {
output = new PrintStream(new OutputStreamDelegator(System.out, new FileOutputStream(reportFile)));
} catch (final IOException e) {
System.err.println(e);
}
}
return output;
}
public static final class ScriptRunnable extends AbstractScriptRunnable implements Callable<ScriptRunnable.Result> {
private final Result result = new Result();
public class Result {
private boolean passed = true;
public String expected;
public String out;
public String err;
public Throwable exception;
public ScriptRunnable getTest() {
return ScriptRunnable.this;
}
public boolean passed() {
return passed;
}
@Override
public String toString() {
return getTest().toString();
}
}
public ScriptRunnable(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> scriptArguments) {
super(framework, testFile, engineOptions, testOptions, scriptArguments);
}
@Override
protected void log(final String msg) {
System.err.println(msg);
}
@Override
protected void fail(final String message) {
throw new TestFailedError(message);
}
@Override
protected void compile() throws IOException {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
final ByteArrayOutputStream err = new ByteArrayOutputStream();
final List<String> args = getCompilerArgs();
int errors;
try {
errors = evaluateScript(out, err, args.toArray(new String[0]));
} catch (final AssertionError e) {
final PrintWriter writer = new PrintWriter(err);
e.printStackTrace(writer);
writer.flush();
errors = 1;
}
if (errors != 0 || checkCompilerMsg) {
result.err = err.toString();
if (expectCompileFailure || checkCompilerMsg) {
final PrintStream outputDest = new PrintStream(new FileOutputStream(getErrorFileName()));
TestHelper.dumpFile(outputDest, new StringReader(new String(err.toByteArray())));
outputDest.println("--");
}
if (errors != 0 && !expectCompileFailure) {
fail(String.format("%d errors compiling %s", errors, testFile));
}
if (checkCompilerMsg) {
compare(getErrorFileName(), expectedFileName, true);
}
}
if (expectCompileFailure && errors == 0) {
fail(String.format("No errors encountered compiling negative test %s", testFile));
}
}
@Override
protected void execute() {
final List<String> args = getRuntimeArgs();
final ByteArrayOutputStream out = new ByteArrayOutputStream();
final ByteArrayOutputStream err = new ByteArrayOutputStream();
try {
final int errors = evaluateScript(out, err, args.toArray(new String[0]));
if (errors != 0 || err.size() > 0) {
if (expectRunFailure) {
return;
}
if (!ignoreStdError) {
try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {
outputFile.write(out.toByteArray());
errorFile.write(err.toByteArray());
}
result.out = out.toString();
result.err = err.toString();
fail(err.toString());
}
}
if (compare) {
final File expectedFile = new File(expectedFileName);
try {
BufferedReader expected;
if (expectedFile.exists()) {
expected = new BufferedReader(new FileReader(expectedFile));
} else {
expected = new BufferedReader(new StringReader(""));
}
compare(new BufferedReader(new StringReader(out.toString())), expected, false);
} catch (final Throwable ex) {
if (expectedFile.exists()) {
copyExpectedFile();
}
try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {
outputFile.write(out.toByteArray());
errorFile.write(err.toByteArray());
}
ex.printStackTrace();
throw ex;
}
}
} catch (final IOException e) {
if (!expectRunFailure) {
fail("Failure running test " + testFile + ": " + e.getMessage());
} // else success
}
}
private void compare(final String fileName, final String expected, final boolean compareCompilerMsg) throws IOException {
final File expectedFile = new File(expected);
BufferedReader expectedReader;
if (expectedFile.exists()) {
expectedReader = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFileName)));
} else {
expectedReader = new BufferedReader(new StringReader(""));
}
final BufferedReader actual = new BufferedReader(new InputStreamReader(new FileInputStream(fileName)));
compare(actual, expectedReader, compareCompilerMsg);
}
private void copyExpectedFile() {
if (!new File(expectedFileName).exists()) {
return;
}
// copy expected file overwriting existing file and preserving last
// modified time of source
try {
Files.copy(FileSystems.getDefault().getPath(expectedFileName), FileSystems.getDefault().getPath(getCopyExpectedFileName()), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
} catch (final IOException ex) {
fail("failed to copy expected " + expectedFileName + " to " + getCopyExpectedFileName() + ": " + ex.getMessage());
}
}
@Override
public Result call() {
try {
runTest();
} catch (final Throwable ex) {
result.exception = ex;
result.passed = false;
ex.printStackTrace();
}
return result;
}
private String getOutputFileName() {
buildDir.mkdirs();
return outputFileName;
}
private String getErrorFileName() {
buildDir.mkdirs();
return errorFileName;
}
private String getCopyExpectedFileName() {
buildDir.mkdirs();
return copyExpectedFileName;
}
}
private void suite() throws Exception {
Locale.setDefault(new Locale(""));
System.setOut(outputStream());
final TestFactory<ScriptRunnable> testFactory = new TestFactory<ScriptRunnable>() {
@Override
public ScriptRunnable createTest(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> arguments) {
return new ScriptRunnable(framework, testFile, engineOptions, testOptions, arguments);
}
@Override
public void log(final String msg) {
System.err.println(msg);
}
};
TestFinder.findAllTests(tests, orphans, testFactory);
Collections.sort(tests, new Comparator<ScriptRunnable>() {
@Override
public int compare(final ScriptRunnable o1, final ScriptRunnable o2) {
return o1.testFile.compareTo(o2.testFile);
}
});
}
@SuppressWarnings("resource")
public boolean run() throws IOException {
final int testCount = tests.size();
int passCount = 0;
int doneCount = 0;
System.out.printf("Found %d tests.\n", testCount);
final long startTime = System.nanoTime();
Runtime.getRuntime().addShutdownHook(shutdownHook);
final List<Future<ScriptRunnable.Result>> futures = new ArrayList<>();
for (final ScriptRunnable test : tests) {
futures.add(executor.submit(test));
}
executor.shutdown();
try {
executor.awaitTermination(60, TimeUnit.MINUTES);
} catch (final InterruptedException ex) {
// empty
}
final List<ScriptRunnable.Result> results = new ArrayList<>();
for (final Future<ScriptRunnable.Result> future : futures) {
if (future.isDone()) {
try {
final ScriptRunnable.Result result = future.get();
results.add(result);
doneCount++;
if (result.passed()) {
passCount++;
}
} catch (CancellationException | ExecutionException ex) {
ex.printStackTrace();
} catch (final InterruptedException ex) {
assert false : "should not reach here";
}
}
}
Collections.sort(results, new Comparator<ScriptRunnable.Result>() {
@Override
public int compare(final ScriptRunnable.Result o1, final ScriptRunnable.Result o2) {
return o1.getTest().testFile.compareTo(o2.getTest().testFile);
}
});
boolean hasFailed = false;
final String failedList = System.getProperty(TEST_FAILED_LIST_FILE);
final boolean hasFailedList = failedList != null;
final boolean hadPreviouslyFailingTests = hasFailedList && new File(failedList).length() > 0;
final FileWriter failedFileWriter = hasFailedList ? new FileWriter(failedList) : null;
try {
final PrintWriter failedListWriter = failedFileWriter == null ? null : new PrintWriter(failedFileWriter);
for (final ScriptRunnable.Result result : results) {
if (!result.passed()) {
if (hasFailed == false) {
hasFailed = true;
System.out.println();
System.out.println("FAILED TESTS");
}
System.out.println(result.getTest());
if(failedFileWriter != null) {
failedListWriter.println(result.getTest().testFile.getPath());
}
if (result.exception != null) {
final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString();
System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n");
System.out.print(result.out != null ? result.out : "");
}
}
}
} finally {
if(failedFileWriter != null) {
failedFileWriter.close();
}
}
final double timeElapsed = (System.nanoTime() - startTime) / 1e9; // [s]
System.out.printf("Tests run: %d/%d tests, passed: %d (%.2f%%), failed: %d. Time elapsed: %.0fmin %.0fs.\n", doneCount, testCount, passCount, 100d * passCount / doneCount, doneCount - passCount, timeElapsed / 60, timeElapsed % 60);
System.out.flush();
finishedLatch.countDown();
if (hasFailed) {
throw new AssertionError("TEST FAILED");
}
if(hasFailedList) {
new File(failedList).delete();
}
if(hadPreviouslyFailingTests) {
System.out.println();
System.out.println("Good job on getting all your previously failing tests pass!");
System.out.println("NOW re-running all tests to make sure you haven't caused any NEW test failures.");
System.out.println();
}
return hadPreviouslyFailingTests;
}
public static void main(final String[] args) throws Exception {
parseArgs(args);
while (new ParallelTestRunner().run()) {
//empty
}
}
private static void parseArgs(final String[] args) {
if (args.length > 0) {
String roots = "";
String reportFile = "";
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--roots") && i != args.length - 1) {
roots += args[++i] + " ";
} else if (args[i].equals("--report-file") && i != args.length - 1) {
reportFile = args[++i];
} else if (args[i].equals("--test262")) {
try {
setTest262Properties();
} catch (final IOException ex) {
System.err.println(ex);
}
}
}
if (!roots.isEmpty()) {
System.setProperty(TEST_JS_ROOTS, roots.trim());
}
if (!reportFile.isEmpty()) {
System.setProperty(TEST_JS_REPORT_FILE, reportFile);
}
}
}
private static void setTest262Properties() throws IOException {
System.setProperty(TEST_JS_ROOTS, "test/test262/test/suite/");
System.setProperty(TEST_JS_FRAMEWORK, "test/script/test262.js test/test262/test/harness/framework.js test/test262/test/harness/sta.js");
System.setProperty(TEST_JS_EXCLUDES_FILE, "test/test262/test/config/excludelist.xml");
System.setProperty(TEST_JS_ENABLE_STRICT_MODE, "true");
final Properties projectProperties = new Properties();
projectProperties.load(new FileInputStream("project.properties"));
String excludeList = projectProperties.getProperty("test262-test-sys-prop.test.js.exclude.list", "");
final Pattern pattern = Pattern.compile("\\$\\{([^}]+)}");
for (;;) {
final Matcher matcher = pattern.matcher(excludeList);
if (!matcher.find()) {
break;
}
final String propertyValue = projectProperties.getProperty(matcher.group(1), "");
excludeList = excludeList.substring(0, matcher.start()) + propertyValue + excludeList.substring(matcher.end());
}
System.setProperty(TEST_JS_EXCLUDE_LIST, excludeList);
}
public static final class OutputStreamDelegator extends OutputStream {
private final OutputStream[] streams;
public OutputStreamDelegator(final OutputStream... streams) {
this.streams = streams;
}
@Override
public void write(final int b) throws IOException {
for (final OutputStream stream : streams) {
stream.write(b);
}
}
@Override
public void flush() throws IOException {
for (final OutputStream stream : streams) {
stream.flush();
}
}
}
}
final class TestFailedError extends Error {
private static final long serialVersionUID = 1L;
public TestFailedError(final String message) {
super(message);
}
public TestFailedError(final String message, final Throwable cause) {
super(message, cause);
}
public TestFailedError(final Throwable cause) {
super(cause);
}
}