TestPerformanceDegradationCheck.java

package com.carrotsearch.junitbenchmarks.h2;

import com.carrotsearch.junitbenchmarks.BenchmarkOptions;
import com.carrotsearch.junitbenchmarks.BenchmarkRule;
import org.junit.*;
import org.junit.rules.MethodRule;
import org.junit.runner.JUnitCore;
import org.junit.runner.Result;

import java.io.File;
import java.sql.SQLException;

import static com.carrotsearch.junitbenchmarks.BenchmarkOptionsSystemProperties.*;
import static org.junit.Assert.assertEquals;

/**
 * Test that performance degradation is detected.
 */
public class TestPerformanceDegradationCheck
{
    @Before
    @After
    public void cleanup()
    {
        System.setProperty(PERF_IGNORE_UP_TO_RUN_PROPERTY, "0");
        System.setProperty(PERF_DIFF_TO_AVERAGE_PROPERTY, "-1");
        System.setProperty(PERF_DIFF_TO_LAST_RUN_PROPERTY, "-1");
        System.setProperty(BENCHMARK_ROUNDS_PROPERTY, "");
        System.setProperty(WARMUP_ROUNDS_PROPERTY, "");
        MyTest.dbFileFull.delete();
    }

    @Test
    public void testAnnotations() throws Exception
    {
        {
            MyTest.sleepTime = 100;
            assertEquals(0, JUnitCore.runClasses(MyTestWithAnnotation.class).getFailureCount());
            MyTest.sleepTime = 150;
            final Result result1 = JUnitCore.runClasses(MyTestWithAnnotation.class);
            assertEquals(1, result1.getFailureCount());
            assertEquals(true, result1.getFailures().get(0).getMessage().contains("compared to average"));
            MyTest.sleepTime = 50;
            assertEquals(0, JUnitCore.runClasses(MyTestWithAnnotation.class).getFailureCount());
            MyTest.sleepTime = 90;
            final Result result2 = JUnitCore.runClasses(MyTestWithAnnotation.class);
            assertEquals(1, result2.getFailureCount());
            assertEquals(true, result2.getFailures().get(0).getMessage().contains("compared to last run"));
        }
    }

    @Test
    public void testSystemProperties() throws Exception
    {
        {
            // Speed up tests ...
            System.setProperty(BENCHMARK_ROUNDS_PROPERTY, "1");
            System.setProperty(WARMUP_ROUNDS_PROPERTY, "0");

            // No assertions should be in effect ...
            MyTest.sleepTime = 100;
            assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
            MyTest.sleepTime = 150;
            assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
            MyTest.sleepTime = 50;
            assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
            MyTest.sleepTime = 90;
            assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());

            // Enable diff to last ...
            System.setProperty(PERF_DIFF_TO_LAST_RUN_PROPERTY, "0.1");
            MyTest.sleepTime = 110;
            final Result result1 = JUnitCore.runClasses(MyTest.class);
            assertEquals(1, result1.getFailureCount());
            assertEquals(true, result1.getFailures().get(0).getMessage().contains("compared to last run"));

            // Enable diff to average ...
            System.setProperty(PERF_DIFF_TO_LAST_RUN_PROPERTY, "-1");
            System.setProperty(PERF_DIFF_TO_AVERAGE_PROPERTY, "0.1");
            System.setProperty(PERF_AVERAGE_OVER_RUNS_PROPERTY, "10");
            MyTest.sleepTime = 210;
            final Result result2 = JUnitCore.runClasses(MyTest.class);
            assertEquals(1, result2.getFailureCount());
            assertEquals(true, result2.getFailures().get(0).getMessage().contains("compared to average"));
        }
    }

    @Test
    public void testIgnoreUpToRun() throws Exception
    {
        {
            // Speed up tests ...
            System.setProperty(BENCHMARK_ROUNDS_PROPERTY, "1");
            System.setProperty(WARMUP_ROUNDS_PROPERTY, "0");

            // No assertions should be in effect ...
            for (int a = 0; a < 5; a++)
            {
                MyTest.sleepTime = 100;
                assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
            }

            // Enable diff to last ...
            System.setProperty(PERF_DIFF_TO_LAST_RUN_PROPERTY, "0.1");
            MyTest.sleepTime = 200;
            final Result result1 = JUnitCore.runClasses(MyTest.class);
            assertEquals(1, result1.getFailureCount());
            assertEquals(true, result1.getFailures().get(0).getMessage().contains("compared to last run"));

            System.setProperty(PERF_IGNORE_UP_TO_RUN_PROPERTY, "5");
            assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());

            // Enable diff to average ...
            System.setProperty(PERF_DIFF_TO_LAST_RUN_PROPERTY, "-1");
            System.setProperty(PERF_DIFF_TO_AVERAGE_PROPERTY, "0.1");
            System.setProperty(PERF_AVERAGE_OVER_RUNS_PROPERTY, "10");
            System.setProperty(PERF_IGNORE_UP_TO_RUN_PROPERTY, "0");
            final Result result2 = JUnitCore.runClasses(MyTest.class);
            assertEquals(1, result2.getFailureCount());
            assertEquals(true, result2.getFailures().get(0).getMessage().contains("compared to average"));

            System.setProperty(PERF_IGNORE_UP_TO_RUN_PROPERTY, "6");
            assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
        }
    }

    @Test
    public void testAverageOverRuns() throws Exception
    {
        {
            // Speed up tests ...
            System.setProperty(BENCHMARK_ROUNDS_PROPERTY, "1");
            System.setProperty(WARMUP_ROUNDS_PROPERTY, "0");

            // No assertions should be in effect ...
            for (int a = 0; a < 5; a++)
            {
                MyTest.sleepTime = 100;
                assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
            }
            for (int a = 0; a < 5; a++)
            {
                MyTest.sleepTime = 200;
                assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
            }

            // Enable diff to average ...
            System.setProperty(PERF_DIFF_TO_AVERAGE_PROPERTY, "0.2");
            System.setProperty(PERF_AVERAGE_OVER_RUNS_PROPERTY, "10");
            System.setProperty(PERF_IGNORE_UP_TO_RUN_PROPERTY, "0");
            MyTest.sleepTime = 200;
            final Result result1 = JUnitCore.runClasses(MyTest.class);
            assertEquals(1, result1.getFailureCount());
            assertEquals(true, result1.getFailures().get(0).getMessage().contains("compared to average"));

            System.setProperty(PERF_AVERAGE_OVER_RUNS_PROPERTY, "6");
            assertEquals(0, JUnitCore.runClasses(MyTest.class).getFailureCount());
        }
    }

    public static class MyTest
    {

        static final File dbFile = new File(MyTest.class.getName());
        static final File dbFileFull = new File(dbFile.getName() + ".h2.db");
        static H2Consumer h2consumer;

        static long sleepTime = 100;

        @Rule
        public MethodRule benchmarkRun = new BenchmarkRule(h2consumer);

        @BeforeClass
        public static void checkFile() throws SQLException
        {
            h2consumer = new H2Consumer(dbFile);
        }

        @Test
        public void test() throws Exception
        {
            Thread.sleep(sleepTime);
        }

        @AfterClass
        public static void verify() throws Exception
        {
            h2consumer.close();
        }
    }

    public static class MyTestWithAnnotation extends MyTest
    {

        @Override
        @Test
        @BenchmarkOptions(callgc = false, warmupRounds = 1, benchmarkRounds = 1, perfDiffToAverage = 0.4d, perfDiffToLastRun = 0.6d)
        public void test() throws Exception {
            super.test();
        }
    }
}