/* * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.hazelcast.mapreduce; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.core.ICompletableFuture; import com.hazelcast.spi.NodeEngine; import com.hazelcast.test.HazelcastParallelClassRunner; import com.hazelcast.test.HazelcastTestSupport; import com.hazelcast.test.TestHazelcastInstanceFactory; import com.hazelcast.test.annotation.ParallelTest; import com.hazelcast.test.annotation.QuickTest; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import java.io.IOException; import java.io.Serializable; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @RunWith(HazelcastParallelClassRunner.class) @Category({QuickTest.class, ParallelTest.class}) @Ignore @SuppressWarnings("unused") public class CustomDataSourceMapReduceTest extends HazelcastTestSupport { private static final Comparator<Map.Entry<String, Integer>> ENTRYSET_COMPARATOR = new Comparator<Map.Entry<String, Integer>>() { @Override public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) { Integer i1 = Integer.parseInt(o1.getKey()); Integer i2 = Integer.parseInt(o2.getKey()); return i1.compareTo(i2); } }; @Test(timeout = 60000) public void testMapReduceWithCustomKeyValueSource() throws Exception { TestHazelcastInstanceFactory nodeFactory = createHazelcastInstanceFactory(3); final HazelcastInstance h1 = nodeFactory.newHazelcastInstance(); final HazelcastInstance h2 = nodeFactory.newHazelcastInstance(); final HazelcastInstance h3 = nodeFactory.newHazelcastInstance(); assertClusterSize(3, h1, h3); assertClusterSizeEventually(3, h2); JobTracker jobTracker = h1.getJobTracker("default"); Job<String, Integer> job = jobTracker.newJob(new CustomKeyValueSource()); ICompletableFuture<Map<String, Integer>> completableFuture = job.chunkSize(10).mapper(new CustomMapper()) .combiner(new CustomCombinerFactory()) .reducer(new CustomReducerFactory()).submit(); Map<String, Integer> result = completableFuture.get(); assertEquals(1000, result.size()); List<Map.Entry<String, Integer>> entrySet = new ArrayList(result.entrySet()); Collections.sort(entrySet, ENTRYSET_COMPARATOR); int count = 0; for (Map.Entry<String, Integer> entry : entrySet) { assertEquals(String.valueOf(count), entry.getKey()); assertEquals(count++ * 6, (int) entry.getValue()); } } public static class CustomKeyValueSource extends KeyValueSource<String, Integer> implements Serializable { private transient List<Map.Entry<String, Integer>> entries; private transient Iterator<Map.Entry<String, Integer>> iterator; private transient Map.Entry<String, Integer> nextElement; @Override public boolean open(NodeEngine nodeEngine) { entries = new ArrayList<Map.Entry<String, Integer>>(); for (int i = 0; i < 1000; i++) { entries.add(new AbstractMap.SimpleEntry<String, Integer>(String.valueOf(i), i)); } iterator = entries.iterator(); return true; } @Override public boolean hasNext() { if (iterator == null) { return false; } boolean hasNext = iterator.hasNext(); nextElement = hasNext ? iterator.next() : null; return hasNext; } @Override public String key() { if (nextElement == null) { throw new IllegalStateException("no more elements"); } return nextElement.getKey(); } @Override public Map.Entry<String, Integer> element() { if (nextElement == null) { throw new IllegalStateException("no more elements"); } return nextElement; } @Override public boolean reset() { iterator = null; nextElement = null; entries = new ArrayList<Map.Entry<String, Integer>>(); return true; } @Override public void close() throws IOException { } } public static class CustomMapper implements Mapper<String, Integer, String, Integer> { @Override public void map(String key, Integer value, Context<String, Integer> context) { context.emit(key, value); context.emit(key, value); } } public static class CustomCombiner extends Combiner<Integer, Integer> { private int value; @Override public void combine(Integer value) { this.value += value; } @Override public Integer finalizeChunk() { int oldValue = value; value = 0; return oldValue; } } public static class CustomCombinerFactory implements CombinerFactory<String, Integer, Integer> { @Override public Combiner<Integer, Integer> newCombiner(String key) { return new CustomCombiner(); } } public static class CustomReducer extends Reducer<Integer, Integer> { private int value; @Override public void reduce(Integer value) { this.value += value; } @Override public Integer finalizeReduce() { return value; } } public static class CustomReducerFactory implements ReducerFactory<String, Integer, Integer> { @Override public Reducer<Integer, Integer> newReducer(String key) { return new CustomReducer(); } } }