MapReduceTest.java

/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.mapreduce;

import com.hazelcast.core.ExecutionCallback;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.ICompletableFuture;
import com.hazelcast.core.IMap;
import com.hazelcast.mapreduce.helpers.Employee;
import com.hazelcast.nio.ObjectDataInput;
import com.hazelcast.nio.ObjectDataOutput;
import com.hazelcast.nio.serialization.DataSerializable;
import com.hazelcast.spi.NodeEngine;
import com.hazelcast.test.AssertTask;
import com.hazelcast.test.HazelcastParallelClassRunner;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.TestHazelcastInstanceFactory;
import com.hazelcast.test.annotation.ConfigureParallelRunnerWith;
import com.hazelcast.test.annotation.ParallelTest;
import com.hazelcast.test.annotation.QuickTest;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.io.IOException;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Semaphore;
import java.util.logging.Logger;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

@RunWith(HazelcastParallelClassRunner.class)
@Category({QuickTest.class, ParallelTest.class})
@Ignore
@ConfigureParallelRunnerWith(MapReduceParallelRunnerOptions.class)
public class MapReduceTest extends HazelcastTestSupport {

    private static final int TEST_TIMEOUT = 60000;
    private static final String MAP_NAME = "default";

    private static final Logger LOGGER = Logger.getLogger("test");

    private void tripTerminate(HazelcastInstance... instances) {
        for (HazelcastInstance instance : instances) {
            try {
                instance.getLifecycleService().terminate();
            } catch (Throwable ex) {
                LOGGER.log(java.util.logging.Level.INFO, ex.getMessage(), ex);
            }
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void test_early_finalization_combiner_github_5283() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            KeyValueSource<Integer, Integer> kvs = integerKvSource(m1);
            KeyValueSource<Integer, Integer> wrapper = new MapKeyValueSourceAdapter<Integer, Integer>(kvs);
            Job<Integer, Integer> job = tracker.newJob(wrapper);

            ICompletableFuture<Map<String, List<Integer>>> future =
                    job.mapper(new TestMapper())
                            .combiner(new FinalizingCombinerFactory())
                            .reducer(new ListBasedReducerFactory())
                            .submit();

            Map<String, List<Integer>> result = future.get();

            assertEquals(100, result.size());
            for (List<Integer> value : result.values()) {
                assertEquals(1, value.size());
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testPartitionPostpone() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            KeyValueSource<Integer, Integer> kvs = integerKvSource(m1);
            KeyValueSource<Integer, Integer> wrapper = new MapKeyValueSourceAdapter<Integer, Integer>(kvs);
            Job<Integer, Integer> job = tracker.newJob(wrapper);
            ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TestMapper()).submit();

            Map<String, List<Integer>> result = future.get();

            assertEquals(100, result.size());
            for (List<Integer> value : result.values()) {
                assertEquals(1, value.size());
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void test_collide_user_provided_combiner_list_result_github_3614() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            KeyValueSource<Integer, Integer> kvs = integerKvSource(m1);
            KeyValueSource<Integer, Integer> wrapper = new MapKeyValueSourceAdapter<Integer, Integer>(kvs);
            Job<Integer, Integer> job = tracker.newJob(wrapper);
            ICompletableFuture<Map<String, List<Integer>>> future =
                    job.mapper(new TestMapper())
                            .combiner(new ListResultingCombinerFactory())
                            .reducer(new ListBasedReducerFactory())
                            .submit();

            Map<String, List<Integer>> result = future.get();

            assertEquals(100, result.size());
            for (List<Integer> value : result.values()) {
                assertEquals(1, value.size());
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT, expected = ExecutionException.class)
    public void testExceptionDistributionWithCollator() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, List<Integer>>> future = job
                    .mapper(new ExceptionThrowingMapper())
                    .submit(new Collator<Map.Entry<String, List<Integer>>, Map<String, List<Integer>>>() {
                        @Override
                        public Map<String, List<Integer>> collate(
                                Iterable<Map.Entry<String, List<Integer>>> values) {
                            return null;
                        }
                    });

            try {
                future.get();
                fail();
            } catch (Exception e) {
                e.printStackTrace();
                assertTrue(e.getCause() instanceof NullPointerException);
                throw e;
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT, expected = ExecutionException.class)
    public void testExceptionDistribution() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new ExceptionThrowingMapper()).submit();

            try {
                future.get();
                fail();
            } catch (Exception e) {
                e.printStackTrace();
                assertTrue(e.getCause() instanceof NullPointerException);
                throw e;
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT, expected = CancellationException.class)
    public void testInProcessCancellation() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TimeConsumingMapper()).submit();

            future.cancel(true);

            try {
                future.get();
                fail();
            } catch (Exception e) {
                e.printStackTrace();
                throw e;
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testMapper() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TestMapper()).submit();

            Map<String, List<Integer>> result = future.get();

            assertEquals(100, result.size());
            for (List<Integer> value : result.values()) {
                assertEquals(1, value.size());
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Ignore//https://github.com/hazelcast/hazelcast/issues/6059
    @Test(timeout = TEST_TIMEOUT)
    public void testKeyedMapperCollator() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 10000; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Integer> future = job.onKeys(50).mapper(new TestMapper()).submit(new GroupingTestCollator());

            int result = future.get();

            assertEquals(50, result);
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Ignore//https://github.com/hazelcast/hazelcast/issues/6059
    @Test(timeout = TEST_TIMEOUT)
    public void testKeyPredicateMapperCollator() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 10000; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Integer> future = job
                    .keyPredicate(new TestKeyPredicate())
                    .mapper(new TestMapper())
                    .submit(new GroupingTestCollator());

            int result = future.get();

            assertEquals(50, result);
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testMapperComplexMapping() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new GroupingTestMapper(2)).submit();

            Map<String, List<Integer>> result = future.get();

            assertEquals(1, result.size());
            assertEquals(25, result.values().iterator().next().size());
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testMapperReducer() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, Integer>> future = job
                    .mapper(new GroupingTestMapper())
                    .reducer(new TestReducerFactory())
                    .submit();

            Map<String, Integer> result = future.get();

            // pre-calculate results
            int[] expectedResults = new int[4];
            for (int i = 0; i < 100; i++) {
                int index = i % 4;
                expectedResults[index] += i;
            }

            for (int i = 0; i < 4; i++) {
                assertEquals(expectedResults[i], (int) result.get(String.valueOf(i)));
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testMapperReducerChunked() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 10000; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            JobCompletableFuture<Map<String, Integer>> future = job
                    .chunkSize(10)
                    .mapper(new GroupingTestMapper())
                    .reducer(new TestReducerFactory())
                    .submit();

            TrackableJob trackableJob = tracker.getTrackableJob(future.getJobId());
            final JobProcessInformation processInformation = trackableJob.getJobProcessInformation();
            Map<String, Integer> result = future.get();

            // pre-calculate results
            int[] expectedResults = new int[4];
            for (int i = 0; i < 10000; i++) {
                int index = i % 4;
                expectedResults[index] += i;
            }

            for (int i = 0; i < 4; i++) {
                assertEquals(expectedResults[i], (int) result.get(String.valueOf(i)));
            }

            assertTrueEventually(new AssertTask() {
                @Override
                public void run() {
                    if (processInformation.getProcessedRecords() < 10000) {
                        System.err.println(processInformation.getProcessedRecords());
                    }
                    assertEquals(10000, processInformation.getProcessedRecords());
                }
            });
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testMapperCollator() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).submit(new GroupingTestCollator());

            int result = future.get();

            // pre-calculate result
            int expectedResult = 0;
            for (int i = 0; i < 100; i++) {
                expectedResult += i;
            }

            for (int i = 0; i < 4; i++) {
                assertEquals(expectedResult, result);
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testMapperReducerCollator() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory())
                    .submit(new TestCollator());

            int result = future.get();

            // pre-calculate result
            int expectedResult = 0;
            for (int i = 0; i < 100; i++) {
                expectedResult += i;
            }

            for (int i = 0; i < 4; i++) {
                assertEquals(expectedResult, result);
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testAsyncMapper() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            final Map<String, List<Integer>> listenerResults = new HashMap<String, List<Integer>>();
            final Semaphore semaphore = new Semaphore(1);
            semaphore.acquire();

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, List<Integer>>> future = job.mapper(new TestMapper()).submit();

            future.andThen(new ExecutionCallback<Map<String, List<Integer>>>() {
                @Override
                public void onResponse(Map<String, List<Integer>> response) {
                    try {
                        listenerResults.putAll(response);
                    } finally {
                        semaphore.release();
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    semaphore.release();
                }
            });

            semaphore.acquire();

            assertEquals(100, listenerResults.size());
            for (List<Integer> value : listenerResults.values()) {
                assertEquals(1, value.size());
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testKeyedAsyncMapper() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            final Map<String, List<Integer>> listenerResults = new HashMap<String, List<Integer>>();
            final Semaphore semaphore = new Semaphore(1);
            semaphore.acquire();

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, List<Integer>>> future = job.onKeys(50).mapper(new TestMapper()).submit();

            future.andThen(new ExecutionCallback<Map<String, List<Integer>>>() {
                @Override
                public void onResponse(Map<String, List<Integer>> response) {
                    try {
                        listenerResults.putAll(response);
                    } finally {
                        semaphore.release();
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    semaphore.release();
                }
            });

            semaphore.acquire();

            assertEquals(1, listenerResults.size());
            for (List<Integer> value : listenerResults.values()) {
                assertEquals(1, value.size());
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testAsyncMapperReducer() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {

            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            final Map<String, Integer> listenerResults = new HashMap<String, Integer>();
            final Semaphore semaphore = new Semaphore(1);
            semaphore.acquire();

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Map<String, Integer>> future = job.mapper(new GroupingTestMapper())
                    .reducer(new TestReducerFactory()).submit();

            future.andThen(new ExecutionCallback<Map<String, Integer>>() {
                @Override
                public void onResponse(Map<String, Integer> response) {
                    try {
                        listenerResults.putAll(response);
                    } finally {
                        semaphore.release();
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    semaphore.release();
                }
            });

            // pre-calculate results
            int[] expectedResults = new int[4];
            for (int i = 0; i < 100; i++) {
                int index = i % 4;
                expectedResults[index] += i;
            }

            semaphore.acquire();

            for (int i = 0; i < 4; i++) {
                assertEquals(expectedResults[i], (int) listenerResults.get(String.valueOf(i)));
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testAsyncMapperCollator() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            final int[] result = new int[1];
            final Semaphore semaphore = new Semaphore(1);
            semaphore.acquire();

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).submit(new GroupingTestCollator());

            future.andThen(new ExecutionCallback<Integer>() {
                @Override
                public void onResponse(Integer response) {
                    try {
                        result[0] = response;
                    } finally {
                        semaphore.release();
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    semaphore.release();
                }
            });

            // pre-calculate result
            int expectedResult = 0;
            for (int i = 0; i < 100; i++) {
                expectedResult += i;
            }

            semaphore.acquire();

            for (int i = 0; i < 4; i++) {
                assertEquals(expectedResult, result[0]);
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testAsyncMapperReducerCollator() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            final int[] result = new int[1];
            final Semaphore semaphore = new Semaphore(1);
            semaphore.acquire();

            JobTracker tracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = tracker.newJob(integerKvSource(m1));
            ICompletableFuture<Integer> future = job.mapper(new GroupingTestMapper()).reducer(new TestReducerFactory())
                    .submit(new TestCollator());

            future.andThen(new ExecutionCallback<Integer>() {
                @Override
                public void onResponse(Integer response) {
                    try {
                        result[0] = response;
                    } finally {
                        semaphore.release();
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    semaphore.release();
                }
            });

            // pre-calculate result
            int expectedResult = 0;
            for (int i = 0; i < 100; i++) {
                expectedResult += i;
            }

            semaphore.acquire();

            for (int i = 0; i < 4; i++) {
                assertEquals(expectedResult, result[0]);
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testNullFromObjectCombiner() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        try {
            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker jobTracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = jobTracker.newJob(integerKvSource(m1));
            JobCompletableFuture<Map<String, BigInteger>> future = job.chunkSize(10).mapper(new GroupingTestMapper())
                    .combiner(new ObjectCombinerFactory())
                    .reducer(new ObjectReducerFactory()).submit();

            int[] expectedResults = new int[4];
            for (int i = 0; i < 100; i++) {
                int index = i % 4;
                expectedResults[index] += i;
            }

            Map<String, BigInteger> map = future.get();
            for (int i = 0; i < 4; i++) {
                assertEquals(BigInteger.valueOf(expectedResults[i]), map.get(String.valueOf(i)));
            }
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testNullFromObjectReducer() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        try {
            assertClusterSize(3, h1, h3);
            assertClusterSizeEventually(3, h2);

            IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
            for (int i = 0; i < 100; i++) {
                m1.put(i, i);
            }

            JobTracker jobTracker = h1.getJobTracker("default");
            Job<Integer, Integer> job = jobTracker.newJob(integerKvSource(m1));
            JobCompletableFuture<Map<String, BigInteger>> future = job.chunkSize(10).mapper(new GroupingTestMapper())
                    .combiner(new ObjectCombinerFactory())
                    .reducer(new NullReducerFactory()).submit();

            Map<String, BigInteger> map = future.get();
            assertEquals(0, map.size());
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    @Test(timeout = TEST_TIMEOUT)
    public void testDataSerializableIntermediateObject() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);

        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        assertClusterSize(3, h1, h3);
        assertClusterSizeEventually(3, h2);

        IMap<Integer, Integer> m1 = h1.getMap(MAP_NAME);
        for (int i = 0; i < 100; i++) {
            m1.put(i, i);
        }

        JobTracker jobTracker = h1.getJobTracker("default");
        Job<Integer, Integer> job = jobTracker.newJob(integerKvSource(m1));
        ICompletableFuture<Integer> future = job.mapper(new TestMapper())
                .combiner(new DataSerializableIntermediateCombinerFactory())
                .reducer(new DataSerializableIntermediateReducerFactory())
                .submit(new DataSerializableIntermediateCollator());

        // pre-calculate result
        int expectedResult = 0;
        for (int i = 0; i < 100; i++) {
            expectedResult += i;
        }
        expectedResult = (int) ((double) expectedResult / 100);

        assertEquals(expectedResult, (int) future.get());
    }

    @Test(timeout = TEST_TIMEOUT)
    public void employeeMapReduceTest() throws Exception {
        TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3);
        HazelcastInstance h1 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h2 = nodeFactory.newHazelcastInstance();
        HazelcastInstance h3 = nodeFactory.newHazelcastInstance();

        try {
            IMap<Integer, Employee> map = h1.getMap(randomString());

            int keyCount = 100;
            for (int id = 0; id < keyCount; id++) {
                map.put(id, new Employee(id));
            }

            JobTracker tracker = h1.getJobTracker(randomString());
            Job<Integer, Employee> job = tracker.newJob(KeyValueSource.<Integer, Employee>fromMap(map));

            ICompletableFuture<Map<Integer, Set<Employee>>> future = job
                    .mapper(new ModIdMapper(2))
                    .combiner(new RangeIdCombinerFactory(10, 30))
                    .reducer(new IdReducerFactory(10, 20, 30))
                    .submit();

            Map<Integer, Set<Employee>> result = future.get();

            assertEquals("Expected 8 employees with ids ending 2, 4, 6, 8", 8, result.size());
        } finally {
            tripTerminate(h1, h2, h3);
        }
    }

    static KeyValueSource<Integer, Integer> integerKvSource(IMap<Integer, Integer> m) {
        return KeyValueSource.<Integer, Integer>fromMap(m);
    }

    public static class ModIdMapper implements Mapper<Integer, Employee, Integer, Employee> {

        private int mod = 0;

        public ModIdMapper(int mod) {
            this.mod = mod;
        }

        public void map(Integer key, Employee e, Context<Integer, Employee> context) {
            if (e.getId() % mod == 0) {
                context.emit(key, e);
            }
        }
    }

    public static class RangeIdCombinerFactory implements CombinerFactory<Integer, Employee, Set<Employee>> {

        private int min = 0, max = 0;

        public RangeIdCombinerFactory(int min, int max) {
            this.min = min;
            this.max = max;
        }

        public Combiner<Employee, Set<Employee>> newCombiner(Integer key) {
            return new EmployeeCombiner();
        }

        private class EmployeeCombiner extends Combiner<Employee, Set<Employee>> {
            private Set<Employee> passed = new HashSet<Employee>();

            public void combine(Employee e) {
                if (e.getId() >= min && e.getId() <= max) {
                    passed.add(e);
                }
            }

            public Set<Employee> finalizeChunk() {
                if (passed.isEmpty()) {
                    return null;
                }
                return passed;
            }

            public void reset() {
                passed = new HashSet<Employee>();
            }
        }
    }

    public static class IdReducerFactory implements ReducerFactory<Integer, Set<Employee>, Set<Employee>> {

        private int[] removeIds = null;

        public IdReducerFactory(int... removeIds) {
            this.removeIds = removeIds;
        }

        public Reducer<Set<Employee>, Set<Employee>> newReducer(Integer key) {
            return new EmployeeReducer();
        }

        private class EmployeeReducer extends Reducer<Set<Employee>, Set<Employee>> {

            private volatile Set<Employee> passed = new HashSet<Employee>();

            public void reduce(Set<Employee> set) {
                for (Employee e : set) {
                    boolean add = true;
                    for (int id : removeIds) {
                        if (e.getId() == id) {
                            add = false;
                            break;
                        }
                    }
                    if (add) {
                        passed.add(e);
                    }
                }
            }

            public Set<Employee> finalizeReduce() {
                if (passed.isEmpty()) {
                    return null;
                }
                return passed;
            }
        }
    }

    public static class EmployeeCollator implements Collator<Map.Entry<Integer, Set<Employee>>, Map<Integer, Set<Employee>>> {

        public Map<Integer, Set<Employee>> collate(Iterable<Map.Entry<Integer, Set<Employee>>> values) {
            Map<Integer, Set<Employee>> result = new HashMap<Integer, Set<Employee>>();
            for (Map.Entry<Integer, Set<Employee>> entry : values) {
                for (Employee e : entry.getValue()) {
                    result.put(e.getId(), entry.getValue());
                }
            }
            return result;
        }
    }

    public static class TupleIntInt implements DataSerializable {

        private int count;
        private int amount;

        public TupleIntInt() {
        }

        public TupleIntInt(int count, int amount) {
            this.count = count;
            this.amount = amount;
        }

        @Override
        public void writeData(ObjectDataOutput out) throws IOException {
            out.writeInt(count);
            out.writeInt(amount);
        }

        @Override
        public void readData(ObjectDataInput in) throws IOException {
            count = in.readInt();
            amount = in.readInt();
        }
    }

    public static class ObjectCombinerFactory implements CombinerFactory<String, Integer, BigInteger> {

        @Override
        public Combiner<Integer, BigInteger> newCombiner(String key) {
            return new ObjectCombiner();
        }
    }

    public static class ObjectCombiner extends Combiner<Integer, BigInteger> {

        private BigInteger count;

        @Override
        public void combine(Integer value) {
            count = count == null ? BigInteger.valueOf(value) : count.add(BigInteger.valueOf(value));
        }

        @Override
        public BigInteger finalizeChunk() {
            return count;
        }

        @Override
        public void reset() {
            count = null;
        }
    }

    public static class ObjectReducerFactory implements ReducerFactory<String, BigInteger, BigInteger> {

        @Override
        public Reducer<BigInteger, BigInteger> newReducer(String key) {
            return new ObjectReducer();
        }
    }

    public static class ObjectReducer extends Reducer<BigInteger, BigInteger> {

        private BigInteger count;

        @Override
        public void reduce(BigInteger value) {
            count = count == null ? value : value.add(count);
        }

        @Override
        public BigInteger finalizeReduce() {
            return count;
        }
    }

    public static class DataSerializableIntermediateCombinerFactory implements CombinerFactory<String, Integer, TupleIntInt> {

        @Override
        public Combiner<Integer, TupleIntInt> newCombiner(String key) {
            return new DataSerializableIntermediateCombiner();
        }
    }

    public static class DataSerializableIntermediateCombiner extends Combiner<Integer, TupleIntInt> {

        private int count;
        private int amount;

        @Override
        public void combine(Integer value) {
            count++;
            amount += value;
        }

        @Override
        public TupleIntInt finalizeChunk() {
            int count = this.count;
            int amount = this.amount;
            this.count = 0;
            this.amount = 0;
            return new TupleIntInt(count, amount);
        }
    }

    public static class DataSerializableIntermediateReducerFactory implements ReducerFactory<String, TupleIntInt, TupleIntInt> {

        @Override
        public Reducer<TupleIntInt, TupleIntInt> newReducer(String key) {
            return new DataSerializableIntermediateReducer();
        }
    }

    public static class DataSerializableIntermediateReducer extends Reducer<TupleIntInt, TupleIntInt> {

        private volatile int count;
        private volatile int amount;

        @Override
        public void reduce(TupleIntInt value) {
            count += value.count;
            amount += value.amount;
        }

        @Override
        public TupleIntInt finalizeReduce() {
            return new TupleIntInt(count, amount);
        }
    }

    public static class DataSerializableIntermediateCollator implements Collator<Map.Entry<String, TupleIntInt>, Integer> {

        @Override
        public Integer collate(Iterable<Map.Entry<String, TupleIntInt>> values) {
            int count = 0;
            int amount = 0;
            for (Map.Entry<String, TupleIntInt> value : values) {
                TupleIntInt tuple = value.getValue();
                count += tuple.count;
                amount += tuple.amount;
            }
            return (int) ((double) amount / count);
        }
    }

    public static class ExceptionThrowingMapper implements Mapper<Integer, Integer, String, Integer> {

        @Override
        public void map(Integer key, Integer value, Context<String, Integer> context) {
            throw new NullPointerException("expected NPE");
        }
    }

    public static class TimeConsumingMapper implements Mapper<Integer, Integer, String, Integer> {

        @Override
        public void map(Integer key, Integer value, Context<String, Integer> collector) {
            try {
                Thread.sleep(1000);
            } catch (Exception ignore) {
            }
            collector.emit(String.valueOf(key), value);
        }
    }

    public static class TestKeyPredicate implements KeyPredicate<Integer> {

        @Override
        public boolean evaluate(Integer key) {
            return key == 50;
        }
    }

    public static class TestMapper implements Mapper<Integer, Integer, String, Integer> {

        @Override
        public void map(Integer key, Integer value, Context<String, Integer> collector) {
            collector.emit(String.valueOf(key), value);
        }
    }

    public static class GroupingTestMapper implements Mapper<Integer, Integer, String, Integer> {

        private int moduleKey = -1;

        public GroupingTestMapper() {
        }

        public GroupingTestMapper(int moduleKey) {
            this.moduleKey = moduleKey;
        }

        @Override
        public void map(Integer key, Integer value, Context<String, Integer> collector) {
            if (moduleKey == -1 || (key % 4) == moduleKey) {
                collector.emit(String.valueOf(key % 4), value);
            }
        }
    }

    public static class TestReducer extends Reducer<Integer, Integer> {

        private volatile int sum = 0;

        @Override
        public void reduce(Integer value) {
            sum += value;
        }

        @Override
        public Integer finalizeReduce() {
            return sum;
        }
    }

    public static class TestReducerFactory implements ReducerFactory<String, Integer, Integer> {

        public TestReducerFactory() {
        }

        @Override
        public Reducer<Integer, Integer> newReducer(String key) {
            return new TestReducer();
        }
    }

    public static class GroupingTestCollator implements Collator<Map.Entry<String, List<Integer>>, Integer> {

        @Override
        public Integer collate(Iterable<Map.Entry<String, List<Integer>>> values) {
            int sum = 0;
            for (Map.Entry<String, List<Integer>> entry : values) {
                for (Integer value : entry.getValue()) {
                    sum += value;
                }
            }
            return sum;
        }
    }

    public static class TestCollator implements Collator<Map.Entry<String, Integer>, Integer> {

        @Override
        public Integer collate(Iterable<Map.Entry<String, Integer>> values) {
            int sum = 0;
            for (Map.Entry<String, Integer> entry : values) {
                sum += entry.getValue();
            }
            return sum;
        }
    }

    public static class NullReducerFactory implements ReducerFactory<String, BigInteger, BigInteger> {

        @Override
        public Reducer<BigInteger, BigInteger> newReducer(String key) {
            return new NullReducer();
        }
    }

    public static class NullReducer extends Reducer<BigInteger, BigInteger> {

        @Override
        public void reduce(BigInteger value) {
        }

        @Override
        public BigInteger finalizeReduce() {
            return null;
        }
    }

    public static class MapKeyValueSourceAdapter<K, V> extends KeyValueSource<K, V> implements DataSerializable,
            PartitionIdAware {

        private volatile KeyValueSource<K, V> keyValueSource;
        private int openCount = 0;

        public MapKeyValueSourceAdapter() {
        }

        public MapKeyValueSourceAdapter(KeyValueSource<K, V> keyValueSource) {
            this.keyValueSource = keyValueSource;
        }

        @Override
        public boolean open(NodeEngine nodeEngine) {
            if (openCount < 2) {
                openCount++;
                return false;
            }
            return keyValueSource.open(nodeEngine);
        }

        @Override
        public boolean hasNext() {
            return keyValueSource.hasNext();
        }

        @Override
        public K key() {
            return keyValueSource.key();
        }

        @Override
        public Map.Entry<K, V> element() {
            return keyValueSource.element();
        }

        @Override
        public boolean reset() {
            return keyValueSource.reset();
        }

        @Override
        public boolean isAllKeysSupported() {
            return keyValueSource.isAllKeysSupported();
        }

        @Override
        public Collection<K> getAllKeys0() {
            return keyValueSource.getAllKeys0();
        }

        @Override
        public void close() throws IOException {
            keyValueSource.close();
        }

        @Override
        public void writeData(ObjectDataOutput out) throws IOException {
            out.writeObject(keyValueSource);
        }

        @Override
        public void readData(ObjectDataInput in) throws IOException {
            keyValueSource = in.readObject();
        }

        @Override
        public void setPartitionId(int partitionId) {
            if (keyValueSource instanceof PartitionIdAware) {
                ((PartitionIdAware) keyValueSource).setPartitionId(partitionId);
            }
        }
    }

    public static class ListResultingCombinerFactory implements CombinerFactory<String, Integer, List<Integer>> {

        @Override
        public Combiner<Integer, List<Integer>> newCombiner(String key) {
            return new ListResultingCombiner();
        }

        private class ListResultingCombiner extends Combiner<Integer, List<Integer>> {

            private final List<Integer> result = new ArrayList<Integer>();

            @Override
            public void combine(Integer value) {
                result.add(value);
            }

            @Override
            public List<Integer> finalizeChunk() {
                return new ArrayList<Integer>(result);
            }

            @Override
            public void reset() {
                result.clear();
            }
        }
    }

    public static class ListBasedReducerFactory implements ReducerFactory<String, List<Integer>, List<Integer>> {

        @Override
        public Reducer<List<Integer>, List<Integer>> newReducer(String key) {
            return new ListBasedReducer();
        }

        private class ListBasedReducer extends Reducer<List<Integer>, List<Integer>> {

            private final List<Integer> result = new ArrayList<Integer>();

            @Override
            public void reduce(List<Integer> value) {
                result.addAll(value);
            }

            @Override
            public List<Integer> finalizeReduce() {
                return result;
            }
        }
    }

    public static class FinalizingCombinerFactory implements CombinerFactory<String, Integer, List<Integer>> {

        @Override
        public Combiner<Integer, List<Integer>> newCombiner(String key) {
            return new FinalizingCombiner();
        }

        private class FinalizingCombiner extends Combiner<Integer, List<Integer>> {

            private List<Integer> result = new ArrayList<Integer>();

            @Override
            public void combine(Integer value) {
                result.add(value);
            }

            @Override
            public List<Integer> finalizeChunk() {
                return new ArrayList<Integer>(result);
            }

            @Override
            public void reset() {
                result.clear();
            }

            @Override
            public void finalizeCombine() {
                result = null;
            }
        }
    }
}