/*
* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package combo;
import javax.tools.JavaCompiler;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Stack;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;
/**
* An helper class for defining combinatorial (aka "combo" tests). A combo test is made up of one
* or more 'dimensions' - each of which represent a different axis of the test space. For instance,
* if we wanted to test class/interface declaration, one dimension could be the keyword used for
* the declaration (i.e. 'class' vs. 'interface') while another dimension could be the class/interface
* modifiers (i.e. 'public', 'pachake-private' etc.). A combo test consists in running a test instance
* for each point in the test space; that is, for any combination of the combo test dimension:
* <p>
* 'public' 'class'
* 'public' interface'
* 'package-private' 'class'
* 'package-private' 'interface'
* ...
* <p>
* A new test instance {@link ComboInstance} is created, and executed, after its dimensions have been
* initialized accordingly. Each instance can either pass, fail or throw an unexpected error; this helper
* class defines several policies for how failures should be handled during a combo test execution
* (i.e. should errors be ignored? Do we want the first failure to result in a failure of the whole
* combo test?).
* <p>
* Additionally, this helper class allows to specify filter methods that can be used to throw out
* illegal combinations of dimensions - for instance, in the example above, we might want to exclude
* all combinations involving 'protected' and 'private' modifiers, which are disallowed for toplevel
* declarations.
* <p>
* While combo tests can be used for a variety of workloads, typically their main task will consist
* in performing some kind of javac compilation. For this purpose, this framework defines an optimized
* javac context {@link ReusableContext} which can be shared across multiple combo instances,
* when the framework detects it's safe to do so. This allows to reduce the overhead associated with
* compiler initialization when the test space is big.
*/
public class ComboTestHelper<X extends ComboInstance<X>> {
/** Failure mode. */
FailMode failMode = FailMode.FAIL_FAST;
/** Ignore mode. */
IgnoreMode ignoreMode = IgnoreMode.IGNORE_NONE;
/** Combo test instance filter. */
Optional<Predicate<X>> optFilter = Optional.empty();
/** Combo test dimensions. */
List<DimensionInfo<?>> dimensionInfos = new ArrayList<>();
/** Combo test stats. */
Info info = new Info();
/** Shared JavaCompiler used across all combo test instances. */
JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
/** Shared file manager used across all combo test instances. */
StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);
/** Shared context used across all combo instances. */
ReusableContext context = new ReusableContext();
/**
* Set failure mode for this combo test.
*/
public ComboTestHelper<X> withFailMode(FailMode failMode) {
this.failMode = failMode;
return this;
}
/**
* Set ignore mode for this combo test.
*/
public ComboTestHelper<X> withIgnoreMode(IgnoreMode ignoreMode) {
this.ignoreMode = ignoreMode;
return this;
}
/**
* Set a filter for combo test instances to be ignored.
*/
public ComboTestHelper<X> withFilter(Predicate<X> filter) {
optFilter = Optional.of(optFilter.map(filter::and).orElse(filter));
return this;
}
/**
* Adds a new dimension to this combo test, with a given name an array of values.
*/
@SafeVarargs
public final <D> ComboTestHelper<X> withDimension(String name, D... dims) {
return withDimension(name, null, dims);
}
/**
* Adds a new dimension to this combo test, with a given name, an array of values and a
* coresponding setter to be called in order to set the dimension value on the combo test instance
* (before test execution).
*/
@SuppressWarnings("unchecked")
@SafeVarargs
public final <D> ComboTestHelper<X> withDimension(String name, DimensionSetter<X, D> setter, D... dims) {
dimensionInfos.add(new DimensionInfo<>(name, dims, setter));
return this;
}
/**
* Adds a new array dimension to this combo test, with a given base name. This allows to specify
* multiple dimensions at once; the names of the underlying dimensions will be generated from the
* base name, using standard array bracket notation - i.e. "DIM[0]", "DIM[1]", etc.
*/
@SafeVarargs
public final <D> ComboTestHelper<X> withArrayDimension(String name, int size, D... dims) {
return withArrayDimension(name, null, size, dims);
}
/**
* Adds a new array dimension to this combo test, with a given base name, an array of values and a
* coresponding array setter to be called in order to set the dimension value on the combo test
* instance (before test execution). This allows to specify multiple dimensions at once; the names
* of the underlying dimensions will be generated from the base name, using standard array bracket
* notation - i.e. "DIM[0]", "DIM[1]", etc.
*/
@SafeVarargs
public final <D> ComboTestHelper<X> withArrayDimension(String name, ArrayDimensionSetter<X, D> setter, int size, D... dims) {
for (int i = 0 ; i < size ; i++) {
dimensionInfos.add(new ArrayDimensionInfo<>(name, dims, i, setter));
}
return this;
}
/**
* Returns the stat object associated with this combo test.
*/
public Info info() {
return info;
}
/**
* Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
* execute a new test instance (built using given supplier) for each such combination.
*/
public void run(Supplier<X> instanceBuilder) {
run(instanceBuilder, null);
}
/**
* Runs this combo test. This will generate the combinatorial explosion of all dimensions, and
* execute a new test instance (built using given supplier) for each such combination. Before
* executing the test instance entry point, the supplied initialization method is called on
* the test instance; this is useful for ad-hoc test instance initialization once all the dimension
* values have been set.
*/
public void run(Supplier<X> instanceBuilder, Consumer<X> initAction) {
runInternal(0, new Stack<>(), instanceBuilder, Optional.ofNullable(initAction));
end();
}
/**
* Generate combinatorial explosion of all dimension values and create a new test instance
* for each combination.
*/
@SuppressWarnings({"unchecked", "rawtypes"})
private void runInternal(int index, Stack<DimensionBinding<?>> bindings, Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction) {
if (index == dimensionInfos.size()) {
runCombo(instanceBuilder, initAction, bindings);
} else {
DimensionInfo<?> dinfo = dimensionInfos.get(index);
for (Object d : dinfo.dims) {
bindings.push(new DimensionBinding(d, dinfo));
runInternal(index + 1, bindings, instanceBuilder, initAction);
bindings.pop();
}
}
}
/**
* Run a new test instance using supplied dimension bindings. All required setters and initialization
* method are executed before calling the instance main entry point. Also checks if the instance
* is compatible with the specified test filters; if not, the test is simply skipped.
*/
@SuppressWarnings("unchecked")
private void runCombo(Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction, List<DimensionBinding<?>> bindings) {
X x = instanceBuilder.get();
for (DimensionBinding<?> binding : bindings) {
binding.init(x);
}
initAction.ifPresent(action -> action.accept(x));
info.comboCount++;
if (!optFilter.isPresent() || optFilter.get().test(x)) {
x.run(new Env(bindings));
if (failMode.shouldStop(ignoreMode, info)) {
end();
}
} else {
info.skippedCount++;
}
}
/**
* This method is executed upon combo test completion (either normal or erroneous). Closes down
* all pending resources and dumps useful stats info.
*/
private void end() {
try {
fm.close();
if (info.hasFailures()) {
throw new AssertionError("Failure when executing combo:" + info.lastFailure.orElse(""));
} else if (info.hasErrors()) {
throw new AssertionError("Unexpected exception while executing combo", info.lastError.get());
}
} catch (IOException ex) {
throw new AssertionError("Failure when closing down shared file manager; ", ex);
} finally {
info.dump();
}
}
/**
* Functional interface for specifying combo test instance setters.
*/
public interface DimensionSetter<X extends ComboInstance<X>, D> {
void set(X x, D d);
}
/**
* Functional interface for specifying combo test instance array setters. The setter method
* receives an extra argument for the index of the array element to be set.
*/
public interface ArrayDimensionSetter<X extends ComboInstance<X>, D> {
void set(X x, D d, int index);
}
/**
* Dimension descriptor; each dimension has a name, an array of value and an optional setter
* to be called on the associated combo test instance.
*/
class DimensionInfo<D> {
String name;
D[] dims;
boolean isParameter;
Optional<DimensionSetter<X, D>> optSetter;
DimensionInfo(String name, D[] dims, DimensionSetter<X, D> setter) {
this.name = name;
this.dims = dims;
this.optSetter = Optional.ofNullable(setter);
this.isParameter = dims[0] instanceof ComboParameter;
}
}
/**
* Array dimension descriptor. The dimension name is derived from a base name and an index using
* standard bracket notation; ; the setter accepts an additional 'index' argument to point
* to the array element to be initialized.
*/
class ArrayDimensionInfo<D> extends DimensionInfo<D> {
public ArrayDimensionInfo(String name, D[] dims, int index, ArrayDimensionSetter<X, D> setter) {
super(String.format("%s[%d]", name, index), dims,
setter != null ? (x, d) -> setter.set(x, d, index) : null);
}
}
/**
* Failure policies for a combo test run.
*/
public enum FailMode {
/** Combo test fails when first failure is detected. */
FAIL_FAST,
/** Combo test fails after all instances have been executed. */
FAIL_AFTER;
boolean shouldStop(IgnoreMode ignoreMode, Info info) {
switch (this) {
case FAIL_FAST:
return !ignoreMode.canIgnore(info);
default:
return false;
}
}
}
/**
* Ignore policies for a combo test run.
*/
public enum IgnoreMode {
/** No error or failure is ignored. */
IGNORE_NONE,
/** Only errors are ignored. */
IGNORE_ERRORS,
/** Only failures are ignored. */
IGNORE_FAILURES,
/** Both errors and failures are ignored. */
IGNORE_ALL;
boolean canIgnore(Info info) {
switch (this) {
case IGNORE_ERRORS:
return info.failCount == 0;
case IGNORE_FAILURES:
return info.errCount == 0;
case IGNORE_ALL:
return true;
default:
return info.failCount == 0 && info.errCount == 0;
}
}
}
/**
* A dimension binding. This is essentially a pair of a dimension value and its corresponding
* dimension info.
*/
class DimensionBinding<D> {
D d;
DimensionInfo<D> info;
DimensionBinding(D d, DimensionInfo<D> info) {
this.d = d;
this.info = info;
}
void init(X x) {
info.optSetter.ifPresent(setter -> setter.set(x, d));
}
public String toString() {
return String.format("(%s -> %s)", info.name, d);
}
}
/**
* This class is used to keep track of combo tests stats; info such as numbero of failures/errors,
* number of times a context has been shared/dropped are all recorder here.
*/
public static class Info {
int failCount;
int errCount;
int passCount;
int comboCount;
int skippedCount;
int ctxReusedCount;
int ctxDroppedCount;
Optional<String> lastFailure = Optional.empty();
Optional<Throwable> lastError = Optional.empty();
void dump() {
System.err.println(String.format("%d total checks executed", comboCount));
System.err.println(String.format("%d successes found", passCount));
System.err.println(String.format("%d failures found", failCount));
System.err.println(String.format("%d errors found", errCount));
System.err.println(String.format("%d skips found", skippedCount));
System.err.println(String.format("%d contexts shared", ctxReusedCount));
System.err.println(String.format("%d contexts dropped", ctxDroppedCount));
}
public boolean hasFailures() {
return failCount != 0;
}
public boolean hasErrors() {
return errCount != 0;
}
}
/**
* THe execution environment for a given combo test instance. An environment contains the
* bindings for all the dimensions, along with the combo parameter cache (this is non-empty
* only if one or more dimensions are subclasses of the {@code ComboParameter} interface).
*/
class Env {
List<DimensionBinding<?>> bindings;
Map<String, ComboParameter> parametersCache = new HashMap<>();
@SuppressWarnings({"Unchecked", "rawtypes"})
Env(List<DimensionBinding<?>> bindings) {
this.bindings = bindings;
for (DimensionBinding<?> binding : bindings) {
if (binding.info.isParameter) {
parametersCache.put(binding.info.name, (ComboParameter)binding.d);
};
}
}
Info info() {
return ComboTestHelper.this.info();
}
StandardJavaFileManager fileManager() {
return fm;
}
JavaCompiler javaCompiler() {
return comp;
}
ReusableContext context() {
return context;
}
ReusableContext setContext(ReusableContext context) {
return ComboTestHelper.this.context = context;
}
}
}