TestMutationTesting.java

/*
 * Copyright 2010 Henry Coles
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */
package org.pitest.mutationtest;

import static org.junit.Assert.assertEquals;
import static org.pitest.mutationtest.DetectionStatus.KILLED;
import static org.pitest.mutationtest.DetectionStatus.MEMORY_ERROR;
import static org.pitest.mutationtest.DetectionStatus.NON_VIABLE;
import static org.pitest.mutationtest.DetectionStatus.NO_COVERAGE;
import static org.pitest.mutationtest.DetectionStatus.SURVIVED;
import static org.pitest.mutationtest.DetectionStatus.TIMED_OUT;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Set;

import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.MockitoAnnotations;
import org.pitest.SystemTest;
import org.pitest.classinfo.ClassInfo;
import org.pitest.classinfo.ClassName;
import org.pitest.classpath.ClassloaderByteArraySource;
import org.pitest.classpath.CodeSource;
import org.pitest.classpath.PathFilter;
import org.pitest.classpath.ProjectClassPaths;
import org.pitest.coverage.CoverageDatabase;
import org.pitest.coverage.CoverageGenerator;
import org.pitest.coverage.execute.CoverageOptions;
import org.pitest.coverage.execute.DefaultCoverageGenerator;
import org.pitest.coverage.export.NullCoverageExporter;
import org.pitest.functional.FCollection;
import org.pitest.functional.predicate.False;
import org.pitest.functional.predicate.Predicate;
import org.pitest.functional.prelude.Prelude;
import org.pitest.mutationtest.build.DefaultGrouper;
import org.pitest.mutationtest.build.DefaultTestPrioritiser;
import org.pitest.mutationtest.build.MutationAnalysisUnit;
import org.pitest.mutationtest.build.MutationSource;
import org.pitest.mutationtest.build.MutationTestBuilder;
import org.pitest.mutationtest.build.PercentAndConstantTimeoutStrategy;
import org.pitest.mutationtest.build.WorkerFactory;
import org.pitest.mutationtest.config.DefaultDependencyPathPredicate;
import org.pitest.mutationtest.config.ReportOptions;
import org.pitest.mutationtest.engine.MutationEngine;
import org.pitest.mutationtest.engine.gregor.MethodMutatorFactory;
import org.pitest.mutationtest.engine.gregor.config.GregorEngineFactory;
import org.pitest.mutationtest.engine.gregor.config.Mutator;
import org.pitest.mutationtest.execute.MutationAnalysisExecutor;
import org.pitest.mutationtest.filter.UnfilteredMutationFilter;
import org.pitest.mutationtest.tooling.JarCreatingJarFinder;
import org.pitest.process.DefaultJavaExecutableLocator;
import org.pitest.process.JavaAgent;
import org.pitest.process.LaunchOptions;
import org.pitest.simpletest.ConfigurationForTesting;
import org.pitest.simpletest.TestAnnotationForTesting;
import org.pitest.testapi.Configuration;
import org.pitest.util.Functions;
import org.pitest.util.IsolationUtils;
import org.pitest.util.Timings;

import com.example.MutationsInNestedClasses;
import com.example.MutationsInNestedClassesTest;

@Category(SystemTest.class)
public class TestMutationTesting {

  private MutationAnalysisExecutor mae;
  private Configuration            config;

  private MetaDataExtractor        metaDataExtractor;

  @Before
  public void setUp() {
    MockitoAnnotations.initMocks(this);
    this.config = new ConfigurationForTesting();
    this.metaDataExtractor = new MetaDataExtractor();
    this.mae = new MutationAnalysisExecutor(1,
        Collections
            .<MutationResultListener> singletonList(this.metaDataExtractor));
  }

  public static class NoMutations {

  }

  public static class OneMutationOnly {
    public static int returnOne() {
      return 1;
    }
  }

  public static class ThreeMutations {
    public static int returnOne() {
      return 1;
    }

    public static int returnTwo() {
      return 2;
    }

    public static int returnThree() {
      return 3;
    }
  }

  public static class OneMutationFullTest {
    @TestAnnotationForTesting
    public void testReturnOne() {
      assertEquals(1, OneMutationOnly.returnOne());
    }
  }

  @Test
  public void shouldKillAllCoveredMutations() {
    run(OneMutationOnly.class, OneMutationFullTest.class,
        Mutator.byName("RETURN_VALS"));
    verifyResults(KILLED);
  }

  public static class ThreeMutationsTwoMeaningfullTests {
    @TestAnnotationForTesting
    public void testReturnOne() {
      assertEquals(1, ThreeMutations.returnOne());
    }

    @TestAnnotationForTesting
    public void testReturnTwo() {
      assertEquals(2, ThreeMutations.returnTwo());
    }

    @TestAnnotationForTesting
    public void coverButDoNotTestReturnThree() {
      ThreeMutations.returnThree();
    }
  }

  @Test
  public void shouldDetectedMixOfSurvivingAndKilledMutations() {
    run(ThreeMutations.class, ThreeMutationsTwoMeaningfullTests.class,
        Mutator.byName("RETURN_VALS"));
    verifyResults(SURVIVED, KILLED, KILLED);
  }

  public static class FailingTest {
    @TestAnnotationForTesting
    public void fail() {
      assertEquals(1, 2);
    }
  }

  public static class NoMutationsTest {
    @TestAnnotationForTesting
    public void pass() {

    }
  }

  @Test
  public void shouldReportNoResultsIfNoMutationsPossible() {
    run(NoMutations.class, NoMutationsTest.class, Mutator.byName("RETURN_VALS"));
    verifyResults();
  }

  public static class NoTests {

  }

  @Test
  public void shouldReportStatusOfNoCoverageWhenNoTestsAvailable() {
    run(ThreeMutations.class, NoTests.class, Mutator.byName("RETURN_VALS"));
    verifyResults(NO_COVERAGE, NO_COVERAGE, NO_COVERAGE);
  }

  public static class OneMutationTest {

  }

  public static class InfiniteLoop {
    public static int loop() {
      int i = 1;
      do {
        i++;
        try {
          Thread.sleep(1);
        } catch (final InterruptedException e) {
          e.printStackTrace();
        }
      } while (i < 1);
      i++;
      return i;
    }
  }

  public static class InfiniteLoopTest {
    @TestAnnotationForTesting()
    public void pass() {
      assertEquals(3, InfiniteLoop.loop());
    }
  }

  @Test(timeout = 30000)
  public void shouldDetectAndEscapeFromInfiniteLoopsCausedByMutations() {
    run(InfiniteLoop.class, InfiniteLoopTest.class,
        Mutator.byName("INCREMENTS"));
    verifyResults(KILLED, TIMED_OUT);
  }

  public static class OneMutationFullTestWithSystemPropertyDependency {
    @TestAnnotationForTesting
    public void testReturnOne() {
      if (System.getProperty("foo").equals("foo")) {
        assertEquals(1, OneMutationOnly.returnOne());
      }
    }
  }

  @Test
  public void shouldExportSystemPropertiesToMinionProcess() {
    // System.setProperty("foo", "foo");
    // note surefire is configured to launch this test with -Dfoo=foo
    run(OneMutationOnly.class,
        OneMutationFullTestWithSystemPropertyDependency.class,
        Mutator.byName("RETURN_VALS"));
    verifyResults(KILLED);
  }

  public static class UnviableMutationsTest {
    @TestAnnotationForTesting
    public void test() {
      new OneMutationOnly();
      OneMutationOnly.returnOne();
    }
  }

  @Test
  public void shouldDetectUnviableMutations() {
    run(OneMutationOnly.class, UnviableMutationsTest.class,
        Collections.singleton(new UnviableClassMutator()));
    verifyResults(NON_VIABLE, NON_VIABLE);

  }

  public static class EatsMemoryWhenMutated {
    public static int loop() throws InterruptedException {
      int i = 1;
      final List<String[]> vals = new ArrayList<String[]>();
      Thread.sleep(1500);
      do {
        i++;
        vals.add(new String[9999999]);
        vals.add(new String[9999999]);
        vals.add(new String[9999999]);
        vals.add(new String[9999999]);
      } while (i < 1);
      i++;
      return i;
    }
  }

  public static class EatsMemoryTest {
    @TestAnnotationForTesting()
    public void pass() throws InterruptedException {
      assertEquals(3, EatsMemoryWhenMutated.loop());
    }
  }

  @Ignore("flakey")
  @Test(timeout = 30000)
  public void shouldRecoverFromOutOfMemoryError() {
    run(EatsMemoryWhenMutated.class, EatsMemoryTest.class,
        Mutator.byName("INCREMENTS"));
    verifyResults(KILLED, MEMORY_ERROR);
  }

  @Test
  public void shouldIsolateMutationsFromNestedClasses() {
    // see http://code.google.com/p/pitestrunner/issues/detail?id=17 for full
    // description of this issue
    run(MutationsInNestedClasses.class, MutationsInNestedClassesTest.class,
        Mutator.byName("RETURN_VALS"));
    verifyResults(SURVIVED, SURVIVED);
  }

  @Test
  @Ignore("too brittle")
  public void shouldRecordCorrectLineNumberForMutations() {
    run(OneMutationOnly.class, OneMutationFullTest.class,
        Mutator.byName("RETURN_VALS"));
    verifyLineNumbers(111);
  }

  private void run(final Class<?> clazz, final Class<?> test,
      final Collection<? extends MethodMutatorFactory> mutators) {

    final ReportOptions data = new ReportOptions();

    final Set<Predicate<String>> tests = Collections.singleton(Prelude
        .isEqualTo(test.getName()));
    data.setTargetTests(tests);
    data.setDependencyAnalysisMaxDistance(-1);

    final Set<Predicate<String>> mutees = Collections.singleton(Functions
        .startsWith(clazz.getName()));
    data.setTargetClasses(mutees);

    data.setTimeoutConstant(PercentAndConstantTimeoutStrategy.DEFAULT_CONSTANT);
    data.setTimeoutFactor(PercentAndConstantTimeoutStrategy.DEFAULT_FACTOR);

    final JavaAgent agent = new JarCreatingJarFinder();

    try {
      createEngineAndRun(data, agent, mutators);
    } finally {
      agent.close();
    }
  }

  private void createEngineAndRun(final ReportOptions data,
      final JavaAgent agent,
      final Collection<? extends MethodMutatorFactory> mutators) {

    // data.setConfiguration(this.config);
    final CoverageOptions coverageOptions = createCoverageOptions(data);

    final LaunchOptions launchOptions = new LaunchOptions(agent,
        new DefaultJavaExecutableLocator(), data.getJvmArgs(),
        new HashMap<String, String>());

    final PathFilter pf = new PathFilter(
        Prelude.not(new DefaultDependencyPathPredicate()),
        Prelude.not(new DefaultDependencyPathPredicate()));
    final ProjectClassPaths cps = new ProjectClassPaths(data.getClassPath(),
        data.createClassesFilter(), pf);

    final Timings timings = new Timings();
    final CodeSource code = new CodeSource(cps, coverageOptions.getPitConfig()
        .testClassIdentifier());

    final CoverageGenerator coverageGenerator = new DefaultCoverageGenerator(
        null, coverageOptions, launchOptions, code, new NullCoverageExporter(),
        timings, false);

    final CoverageDatabase coverageData = coverageGenerator.calculateCoverage();

    final Collection<ClassName> codeClasses = FCollection.map(code.getCode(),
        ClassInfo.toClassName());

    final MutationEngine engine = new GregorEngineFactory()
    .createEngineWithMutators(false, False.<String> instance(),
        Collections.<String> emptyList(), mutators, true);

    final MutationConfig mutationConfig = new MutationConfig(engine,
        launchOptions);

    final ClassloaderByteArraySource bas = new ClassloaderByteArraySource(
        IsolationUtils.getContextClassLoader());
    final MutationSource source = new MutationSource(mutationConfig,
        UnfilteredMutationFilter.INSTANCE, new DefaultTestPrioritiser(
            coverageData), bas);

    final WorkerFactory wf = new WorkerFactory(null,
        coverageOptions.getPitConfig(), mutationConfig,
        new PercentAndConstantTimeoutStrategy(data.getTimeoutFactor(),
            data.getTimeoutConstant()), data.isVerbose(), data.getClassPath()
            .getLocalClassPath());

    final MutationTestBuilder builder = new MutationTestBuilder(wf,
        new NullAnalyser(), source, new DefaultGrouper(0));

    final List<MutationAnalysisUnit> tus = builder
        .createMutationTestUnits(codeClasses);

    this.mae.run(tus);
  }

  private CoverageOptions createCoverageOptions(ReportOptions data) {
    return new CoverageOptions(data.getTargetClassesFilter(), this.config,
        data.isVerbose(), data.getDependencyAnalysisMaxDistance());
  }

  protected void verifyResults(final DetectionStatus... detectionStatus) {
    final List<DetectionStatus> expected = Arrays.asList(detectionStatus);
    final List<DetectionStatus> actual = this.metaDataExtractor
        .getDetectionStatus();

    Collections.sort(expected);
    Collections.sort(actual);

    assertEquals(expected, actual);
  }

  protected void verifyLineNumbers(final Integer... lineNumbers) {
    final List<Integer> expected = Arrays.asList(lineNumbers);
    final List<Integer> actual = this.metaDataExtractor.getLineNumbers();

    Collections.sort(expected);
    Collections.sort(actual);

    assertEquals(expected, actual);
  }

}