PerformanceRunner.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.sling.junit.performance.runner;

import org.apache.sling.junit.performance.impl.*;
import org.junit.runners.BlockJUnit4ClassRunner;
import org.junit.runners.model.FrameworkField;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.Statement;

import java.util.ArrayList;
import java.util.List;

/**
 * Custom runner to execute performance tests using JUnit.
 * <p/>
 * For a method to be executed as a performance test, it must be annotated with {@link PerformanceTest}. Every time this
 * annotation is specified, the user must also specify the warm up and execution strategy, because these information are
 * mandatory for the runner to work properly. The warm up and execution strategy can be provided in two ways: by
 * specifying the number of executions to run, or by specifying the amount of time the method should run.
 * <p/>
 * The runner can also invoke one or more {@link Listener}. The listener is specified as a static variable of the test
 * class or as the result of a static method. The listeners are made available to the runner by annotating them with the
 * {@link Listen} annotation.
 * <p/>
 * The runner support lifecycle methods which are executed at various stages of the performance test. These methods are
 * annotated with {@link BeforePerformanceIteration}, {@link AfterPerformanceIteration}, {@link BeforePerformanceTest},
 * {@link AfterPerformanceTest}, {@link BeforeWarmUpIteration}, {@link AfterWarmUpIteration}, {@link BeforeWarmUp} and
 * {@link AfterWarmUp}. Every other standard JUnit annotation is also supported.
 */
public class PerformanceRunner extends BlockJUnit4ClassRunner {

    private Listeners listeners;

    public PerformanceRunner(Class<?> testClass) throws InitializationError {
        super(testClass);

        try {
            listeners = new Listeners(getListeners());
        } catch (Throwable e) {
            throw new InitializationError(e);
        }
    }

    @Override
    protected void collectInitializationErrors(List<Throwable> errors) {
        super.collectInitializationErrors(errors);

        validatePublicVoidNoArgMethods(PerformanceTest.class, false, errors);
        validatePublicVoidNoArgMethods(BeforeWarmUpIteration.class, false, errors);
        validatePublicVoidNoArgMethods(AfterWarmUpIteration.class, false, errors);
        validatePublicVoidNoArgMethods(BeforeWarmUp.class, false, errors);
        validatePublicVoidNoArgMethods(AfterWarmUp.class, false, errors);
        validatePublicVoidNoArgMethods(BeforePerformanceIteration.class, false, errors);
        validatePublicVoidNoArgMethods(AfterPerformanceIteration.class, false, errors);
        validatePublicVoidNoArgMethods(BeforePerformanceTest.class, false, errors);
        validatePublicVoidNoArgMethods(AfterPerformanceTest.class, false, errors);
        validatePerformanceTestsExecutionStrategy(errors);
        validateListenMethodsReturnType(errors);
        validateListenMethodsStatic(errors);
        validateListenMethodPublic(errors);
        validateListenFieldsType(errors);
        validateListenFieldsStatic(errors);
        validateListenFieldPublic(errors);
    }

    private void validatePerformanceTestsExecutionStrategy(List<Throwable> errors) {
        for (FrameworkMethod method : getTestClass().getAnnotatedMethods(PerformanceTest.class)) {
            int warmUpInvocations = getWarmUpInvocations(method);
            int warmUpTime = getWarmUpTime(method);

            if (warmUpInvocations <= 0 && warmUpTime <= 0) {
                errors.add(new Error("Method " + method.getName() + "() should provide a valid warmUpInvocations or warmUpTime"));
            }

            if (warmUpInvocations > 0 && warmUpTime > 0) {
                errors.add(new Error("Method " + method.getName() + "() provides both a valid warmUpInvocations and a warmUpTime"));
            }

            int runInvocations = getRunInvocations(method);
            int runTime = getRunTime(method);

            if (runInvocations <= 0 && runTime <= 0) {
                errors.add(new Error("Method " + method.getName() + "() should provide a valid runInvocations or runTime"));
            }

            if (runInvocations > 0 && runTime > 0) {
                errors.add(new Error("Method " + method.getName() + "() provides both a valid runInvocations or runTime"));
            }
        }
    }

    private void validateListenMethodsReturnType(List<Throwable> errors) {
        for (FrameworkMethod method : getTestClass().getAnnotatedMethods(Listen.class)) {
            if (Listener.class.isAssignableFrom(method.getReturnType())) {
                continue;
            }

            errors.add(new Error("Method " + method.getName() + "() should return an object of type Listener"));
        }
    }

    private void validateListenMethodsStatic(List<Throwable> errors) {
        for (FrameworkMethod method : getTestClass().getAnnotatedMethods(Listen.class)) {
            if (method.isStatic()) {
                continue;
            }

            errors.add(new Error("Method " + method.getName() + "() should be static"));
        }
    }

    private void validateListenMethodPublic(List<Throwable> errors) {
        for (FrameworkMethod method : getTestClass().getAnnotatedMethods(Listen.class)) {
            if (method.isPublic()) {
                continue;
            }

            errors.add(new Error("Method " + method.getName() + "() should be public"));
        }
    }

    private void validateListenFieldsType(List<Throwable> errors) {
        for (FrameworkField field : getTestClass().getAnnotatedFields(Listen.class)) {
            if (Listener.class.isAssignableFrom(field.getType())) {
                continue;
            }

            errors.add(new Error("Field " + field.getName() + " should be of type Listener"));
        }
    }

    private void validateListenFieldsStatic(List<Throwable> errors) {
        for (FrameworkField field : getTestClass().getAnnotatedFields(Listen.class)) {
            if (field.isStatic()) {
                continue;
            }

            errors.add(new Error("Field " + field.getName() + " should be static"));
        }
    }

    private void validateListenFieldPublic(List<Throwable> errors) {
        for (FrameworkField field : getTestClass().getAnnotatedFields(Listen.class)) {
            if (field.isPublic()) {
                continue;
            }

            errors.add(new Error("Field " + field.getName() + " should be public"));
        }
    }

    @Override
    protected List<FrameworkMethod> computeTestMethods() {
        return getTestClass().getAnnotatedMethods(PerformanceTest.class);
    }

    @Override
    protected Statement methodInvoker(FrameworkMethod method, Object test) {
        Statement methodInvoker = super.methodInvoker(method, test);

        Statement invokeWarmUp = methodInvoker;

        invokeWarmUp = withWarmUpIterationStartedEvents(method, test, invokeWarmUp);
        invokeWarmUp = withWarmUpIterationFinishedEvents(method, test, invokeWarmUp);
        invokeWarmUp = withBeforeWarmUpIterations(method, test, invokeWarmUp);
        invokeWarmUp = withAfterWarmUpIterations(method, test, invokeWarmUp);
        invokeWarmUp = withWarmUpIterations(method, test, invokeWarmUp);
        invokeWarmUp = withWarmUpStartedEvents(method, test, invokeWarmUp);
        invokeWarmUp = withWarmUpFinishedEvents(method, test, invokeWarmUp);
        invokeWarmUp = withBeforeWarmUps(method, test, invokeWarmUp);
        invokeWarmUp = withAfterWarmUps(method, test, invokeWarmUp);

        Statement invokePerformanceTest = methodInvoker;

        invokePerformanceTest = withExecutionIterationStartedEvents(method, test, invokePerformanceTest);
        invokePerformanceTest = withExecutionIterationFinishedEvents(method, test, invokePerformanceTest);
        invokePerformanceTest = withBeforePerformanceIterations(method, test, invokePerformanceTest);
        invokePerformanceTest = withAfterPerformanceIterations(method, test, invokePerformanceTest);
        invokePerformanceTest = withPerformanceIterations(method, test, invokePerformanceTest);
        invokePerformanceTest = withExecutionStartedEvents(method, test, invokePerformanceTest);
        invokePerformanceTest = withExecutionFinishedEvents(method, test, invokePerformanceTest);
        invokePerformanceTest = withBeforePerformanceTests(method, test, invokePerformanceTest);
        invokePerformanceTest = withAfterPerformanceTests(method, test, invokePerformanceTest);

        return new RunSerial(invokeWarmUp, invokePerformanceTest);
    }


    protected Statement withBeforeWarmUpIterations(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(BeforeWarmUpIteration.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunBefores(next, methods, test);
    }

    protected Statement withAfterWarmUpIterations(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(AfterWarmUpIteration.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunAfters(next, methods, test);
    }

    protected Statement withWarmUpIterations(FrameworkMethod method, Object test, Statement iteration) {
        return new RunIterations(getWarmUpInvocations(method), getWarmUpTime(method), iteration);
    }

    protected Statement withBeforeWarmUps(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(BeforeWarmUp.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunBefores(next, methods, test);
    }

    protected Statement withAfterWarmUps(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(AfterWarmUp.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunAfters(next, methods, test);
    }

    protected Statement withBeforePerformanceIterations(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(BeforePerformanceIteration.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunBefores(next, methods, test);
    }

    protected Statement withAfterPerformanceIterations(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(AfterPerformanceIteration.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunAfters(next, methods, test);
    }

    protected Statement withPerformanceIterations(FrameworkMethod method, Object test, Statement iteration) {
        return new RunIterations(getRunInvocations(method), getRunTime(method), iteration);
    }

    protected Statement withBeforePerformanceTests(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(BeforePerformanceTest.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunBefores(next, methods, test);
    }

    protected Statement withAfterPerformanceTests(FrameworkMethod method, Object test, Statement next) {
        List<FrameworkMethod> methods = getTestClass().getAnnotatedMethods(AfterPerformanceTest.class);

        if (methods.size() == 0) {
            return next;
        }

        return new RunAfters(next, methods, test);
    }

    protected Statement withWarmUpIterationStartedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunWarmUpIterationStartedEvents(listeners, getTestClass(), method, next);
    }

    protected Statement withWarmUpIterationFinishedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunWarmUpIterationFinishedEvents(listeners, getTestClass(), method, next);
    }

    protected Statement withWarmUpStartedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunWarmUpStartedEvents(listeners, getTestClass(), method, next);
    }

    protected Statement withWarmUpFinishedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunWarmUpFinishedEvents(listeners, getTestClass(), method, next);
    }

    protected Statement withExecutionIterationStartedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunExecutionIterationStartedEvents(listeners, getTestClass(), method, next);
    }

    protected Statement withExecutionIterationFinishedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunExecutionIterationFinishedEvents(listeners, getTestClass(), method, next);
    }

    protected Statement withExecutionStartedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunExecutionStartedEvents(listeners, getTestClass(), method, next);
    }

    protected Statement withExecutionFinishedEvents(FrameworkMethod method, Object test, Statement next) {
        return new RunExecutionFinishedEvents(listeners, getTestClass(), method, next);
    }

    private List<Listener> getListeners() throws Throwable {
        List<Listener> listeners = new ArrayList<Listener>();

        listeners.addAll(readListenersFromStaticFields());
        listeners.addAll(readListenersFromStaticMethods());

        return listeners;
    }

    private List<Listener> readListenersFromStaticMethods() throws Throwable {
        List<Listener> listeners = new ArrayList<Listener>();

        for (FrameworkMethod method : getTestClass().getAnnotatedMethods(Listen.class)) {
            Listener listener = (Listener) method.invokeExplosively(null);

            if (listener == null) {
                throw new IllegalArgumentException("Method " + method.getName() + "() should not return null");
            }

            listeners.add(listener);
        }

        return listeners;
    }

    private List<Listener> readListenersFromStaticFields() throws Exception {
        List<Listener> listeners = new ArrayList<Listener>();

        for (FrameworkField field : getTestClass().getAnnotatedFields(Listen.class)) {
            Listener listener = (Listener) field.get(null);

            if (listener == null) {
                throw new IllegalArgumentException("Field " + field.getName() + " should not be null");
            }

            listeners.add(listener);
        }

        return listeners;
    }

    private int getWarmUpInvocations(FrameworkMethod method) {
        return method.getAnnotation(PerformanceTest.class).warmUpInvocations();
    }

    private int getWarmUpTime(FrameworkMethod method) {
        return method.getAnnotation(PerformanceTest.class).warmUpTime();
    }

    private int getRunInvocations(FrameworkMethod method) {
        return method.getAnnotation(PerformanceTest.class).runInvocations();
    }

    private int getRunTime(FrameworkMethod method) {
        return method.getAnnotation(PerformanceTest.class).runTime();
    }

}