/*
* Copyright (C) 2009 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.coretests;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.Enumeration;
import java.util.Vector;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestFailure;
import junit.framework.TestResult;
import junit.framework.TestSuite;
import dalvik.annotation.AndroidOnly;
import dalvik.annotation.BrokenTest;
import dalvik.annotation.KnownFailure;
import dalvik.annotation.SideEffect;
/**
* A special TestSuite implementation that flattens the hierarchy of a given
* JUnit TestSuite and removes tests after executing them. This is so the core
* tests actually have a chance to succeed, since they do consume quite some
* memory and many tests do not (have a chance to) cleanup properly after
* themselves. The class also implements our filtering mechanism for tests, so
* it becomes easy to only include or exclude tests based on their annotations
* (like, say, execute all Android-only tests that are not known to be broken).
*/
public class CoreTestSuite implements Test {
/**
* Include all normal tests in the suite.
*/
public static final int RUN_NORMAL_TESTS = 1;
/**
* Include all broken tests in the suite.
*/
public static final int RUN_BROKEN_TESTS = 2;
/**
* Include all known failures in the suite.
*/
public static final int RUN_KNOWN_FAILURES = 4;
/**
* Include all Android-only tests in the suite.
*/
public static final int RUN_ANDROID_ONLY = 8;
/**
* Include side-effective tests in the suite.
*/
public static final int RUN_SIDE_EFFECTS = 16;
/**
* Include all tests in the suite.
*/
public static final int RUN_ALL_TESTS =
RUN_NORMAL_TESTS | RUN_BROKEN_TESTS |
RUN_KNOWN_FAILURES | RUN_SIDE_EFFECTS | RUN_ANDROID_ONLY;
/**
* Special treatment for known failures: they are expected to fail, so we
* throw an Exception if they succeed and accept them failing.
*/
public static final int INVERT_KNOWN_FAILURES = 32;
/**
* Run each test in its own VM.
*/
public static final int ISOLATE_ALL = 64;
/**
* Run no test in its own VM.
*/
public static final int ISOLATE_NONE = 128;
/**
* Be verbose.
*/
public static final int VERBOSE = 256;
public static final int REVERSE = 512;
public static final int DRY_RUN = 1024;
private final String name;
/**
* The total number of tests in the original suite.
*/
protected int fTotalCount;
/**
* The number of Android-only tests in the original suite.
*/
protected int fAndroidOnlyCount;
/**
* The number of broken tests in the original suite.
*/
protected int fBrokenCount;
/**
* The number of known failures in the original suite.
*/
protected int fKnownFailureCount;
/**
* The number of side-effective tests in the original suite.
*/
protected int fSideEffectCount;
/**
* The number of normal (non-annotated) tests in the original suite.
*/
protected int fNormalCount;
/**
* The number of ignored tests, that is, the number of tests that were
* excluded from this suite due to their annotations.
*/
protected int fIgnoredCount;
/**
* Contains the actual test cases in a reverse-ordered, flat list.
*/
private Vector<Test> fTests = new Vector<Test>();
private TestCase fVictim;
private int fStep;
private int fFlags;
/**
* Creates a new CoreTestSuite for the given ordinary JUnit Test (which may
* be a TestCase or TestSuite). The CoreTestSuite will be a flattened and
* potentially filtered subset of the original JUnit Test. The flags
* determine the way we filter.
*/
public CoreTestSuite(Test suite, int flags, int step, TestCase victim) {
super();
name = suite.toString();
fStep = step;
addAndFlatten(suite, flags);
fVictim = victim;
fFlags = flags;
}
/**
* Adds the given ordinary JUnit Test (which may be a TestCase or TestSuite)
* to this CoreTestSuite. Note we are storing the tests in reverse order,
* so it's easier to remove a finished test from the end of the list.
*/
private void addAndFlatten(Test test, int flags) {
if (test instanceof TestSuite) {
TestSuite suite = (TestSuite)test;
if ((flags & REVERSE) != 0) {
for (int i = suite.testCount() - 1; i >= 0; i--) {
addAndFlatten(suite.testAt(i), flags);
}
} else {
for (int i = 0; i < suite.testCount(); i++) {
addAndFlatten(suite.testAt(i), flags);
}
}
} else if (test instanceof TestCase) {
TestCase testCase = (TestCase)test;
boolean ignoreMe = false;
boolean isAndroidOnly = hasAnnotation(testCase, AndroidOnly.class);
boolean isBrokenTest = hasAnnotation(testCase, BrokenTest.class);
boolean isKnownFailure = hasAnnotation(testCase, KnownFailure.class);
boolean isSideEffect = hasAnnotation(testCase, SideEffect.class);
boolean isNormalTest =
!(isAndroidOnly || isBrokenTest || isKnownFailure ||
isSideEffect);
if (isAndroidOnly) {
fAndroidOnlyCount++;
}
if (isBrokenTest) {
fBrokenCount++;
}
if (isKnownFailure) {
fKnownFailureCount++;
}
if (isNormalTest) {
fNormalCount++;
}
if (isSideEffect) {
fSideEffectCount++;
}
if ((flags & RUN_ANDROID_ONLY) == 0 && isAndroidOnly) {
ignoreMe = true;
}
if ((flags & RUN_BROKEN_TESTS) == 0 && isBrokenTest) {
ignoreMe = true;
}
if ((flags & RUN_KNOWN_FAILURES) == 0 && isKnownFailure) {
ignoreMe = true;
}
if (((flags & RUN_NORMAL_TESTS) == 0) && isNormalTest) {
ignoreMe = true;
}
if (((flags & RUN_SIDE_EFFECTS) == 0) && isSideEffect) {
ignoreMe = true;
}
this.fTotalCount++;
if (!ignoreMe) {
fTests.add(test);
} else {
this.fIgnoredCount++;
}
} else {
System.out.println("Warning: Don't know how to handle " +
test.getClass().getName() + " " + test.toString());
}
}
/**
* Checks whether the given TestCase class has the given annotation.
*/
@SuppressWarnings("unchecked")
private boolean hasAnnotation(TestCase test, Class clazz) {
try {
Method method = test.getClass().getMethod(test.getName());
return method.getAnnotation(clazz) != null;
} catch (Exception e) {
// Ignore
}
return false;
}
/**
* Runs the tests and collects their result in a TestResult.
*/
public void run(TestResult result) {
// Run tests
int i = 0;
while (fTests.size() != 0 && !result.shouldStop()) {
TestCase test = (TestCase)fTests.elementAt(i);
Thread.currentThread().setContextClassLoader(
test.getClass().getClassLoader());
test.run(result);
/*
if (fVictim != null) {
TestResult dummy = fVictim.run();
if (dummy.failureCount() != 0) {
result.addError(fTests.elementAt(i), new RuntimeException(
"Probable side effect",
((TestFailure)dummy.failures().nextElement()).
thrownException()));
} else if (dummy.errorCount() != 0) {
result.addError(fTests.elementAt(i), new RuntimeException(
"Probable side effect",
((TestFailure)dummy.errors().nextElement()).
thrownException()));
}
}
*/
fTests.remove(i);
if (fTests.size() != 0) {
i = (i + fStep - 1) % fTests.size();
}
}
// Forward overall stats to TestResult, so ResultPrinter can see it.
if (result instanceof CoreTestResult) {
((CoreTestResult)result).updateStats(
fTotalCount, fAndroidOnlyCount, fBrokenCount,
fKnownFailureCount, fNormalCount, fIgnoredCount,
fSideEffectCount);
}
}
/**
* Nulls all reference fields in the given test object. This method helps
* us with those test classes that don't have an explicit tearDown()
* method. Normally the garbage collector should take care of everything,
* but it can't hurt to support it a bit.
*/
private void cleanup(TestCase test) {
Field[] fields = test.getClass().getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
Field f = fields[i];
if (!f.getType().isPrimitive() &&
(f.getModifiers() & Modifier.STATIC) == 0) {
try {
f.setAccessible(true);
f.set(test, null);
} catch (Exception ex) {
// Nothing we can do about it.
}
}
}
}
/**
* Returns the tests as an enumeration. Note this is empty once the tests
* have been executed.
*/
@SuppressWarnings("unchecked")
public Enumeration tests() {
return fTests.elements();
}
/**
* Returns the number of tests in the suite. Note this is zero once the
* tests have been executed.
*/
public int countTestCases() {
return fTests.size();
}
@Override public String toString() {
return name;
}
}