/* * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.hazelcast.cache; import com.hazelcast.cache.CachePartitionLostListenerTest.EventCollectingCachePartitionLostListener; import com.hazelcast.cache.impl.HazelcastServerCachingProvider; import com.hazelcast.cache.impl.event.CachePartitionLostEvent; import com.hazelcast.config.CacheConfig; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.partition.AbstractPartitionLostListenerTest; import com.hazelcast.test.AssertTask; import com.hazelcast.test.HazelcastParametersRunnerFactory; import com.hazelcast.test.annotation.SlowTest; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import javax.cache.Cache; import javax.cache.CacheManager; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; import static com.hazelcast.cache.impl.HazelcastServerCachingProvider.createCachingProvider; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) @Parameterized.UseParametersRunnerFactory(HazelcastParametersRunnerFactory.class) @Category(SlowTest.class) public class CachePartitionLostListenerStressTest extends AbstractPartitionLostListenerTest { @Parameterized.Parameters(name = "numberOfNodesToCrash:{0},withData:{1},nodeLeaveType:{2},shouldExpectPartitionLostEvents:{3}") public static Collection<Object[]> parameters() { return Arrays.asList(new Object[][]{ {1, true, NodeLeaveType.SHUTDOWN, false}, {1, true, NodeLeaveType.TERMINATE, true}, {1, false, NodeLeaveType.SHUTDOWN, false}, {1, false, NodeLeaveType.TERMINATE, true}, {2, true, NodeLeaveType.SHUTDOWN, false}, {2, true, NodeLeaveType.TERMINATE, true}, {2, false, NodeLeaveType.SHUTDOWN, false}, {2, false, NodeLeaveType.TERMINATE, true}, {3, true, NodeLeaveType.SHUTDOWN, false}, {3, true, NodeLeaveType.TERMINATE, true}, {3, false, NodeLeaveType.SHUTDOWN, false}, {3, false, NodeLeaveType.TERMINATE, true} }); } @Parameterized.Parameter(0) public int numberOfNodesToCrash; @Parameterized.Parameter(1) public boolean withData; @Parameterized.Parameter(2) public NodeLeaveType nodeLeaveType; @Parameterized.Parameter(3) public boolean shouldExpectPartitionLostEvents; protected int getNodeCount() { return 5; } protected int getCacheEntryCount() { return 10000; } @Test public void testCachePartitionLostListener() { List<HazelcastInstance> instances = getCreatedInstancesShuffledAfterWarmedUp(); List<HazelcastInstance> survivingInstances = new ArrayList<HazelcastInstance>(instances); List<HazelcastInstance> terminatingInstances = survivingInstances.subList(0, numberOfNodesToCrash); survivingInstances = survivingInstances.subList(numberOfNodesToCrash, instances.size()); HazelcastInstance instance = survivingInstances.get(0); HazelcastServerCachingProvider cachingProvider = createCachingProvider(instance); CacheManager cacheManager = cachingProvider.getCacheManager(); List<EventCollectingCachePartitionLostListener> listeners = registerListeners(cacheManager); if (withData) { for (int i = 0; i < getNodeCount(); i++) { Cache<Integer, Integer> cache = cacheManager.getCache(getIthCacheName(i)); for (int j = 0; j < getCacheEntryCount(); j++) { cache.put(j, j); } } } final String log = "Surviving: " + survivingInstances + " Terminating: " + terminatingInstances; Map<Integer, Integer> survivingPartitions = getMinReplicaIndicesByPartitionId(survivingInstances); stopInstances(terminatingInstances, nodeLeaveType); waitAllForSafeState(survivingInstances); if (shouldExpectPartitionLostEvents) { for (int i = 0; i < getNodeCount(); i++) { assertListenerInvocationsEventually(log, i, numberOfNodesToCrash, listeners.get(i), survivingPartitions); } } else { for (final EventCollectingCachePartitionLostListener listener : listeners) { assertTrueAllTheTime(new AssertTask() { @Override public void run() throws Exception { assertTrue(listener.getEvents().isEmpty()); } }, 1); } } for (int i = 0; i < getNodeCount(); i++) { cacheManager.destroyCache(getIthCacheName(i)); } cacheManager.close(); cachingProvider.close(); } private List<EventCollectingCachePartitionLostListener> registerListeners(CacheManager cacheManager) { CacheConfig<Integer, Integer> config = new CacheConfig<Integer, Integer>(); List<EventCollectingCachePartitionLostListener> listeners = new ArrayList<EventCollectingCachePartitionLostListener>(); for (int i = 0; i < getNodeCount(); i++) { EventCollectingCachePartitionLostListener listener = new EventCollectingCachePartitionLostListener(i); listeners.add(listener); config.setBackupCount(i); Cache<Integer, Integer> cache = cacheManager.createCache(getIthCacheName(i), config); ICache iCache = cache.unwrap(ICache.class); iCache.addPartitionLostListener(listener); } return listeners; } private void assertListenerInvocationsEventually(final String log, final int index, final int numberOfNodesToCrash, final EventCollectingCachePartitionLostListener listener, final Map<Integer, Integer> survivingPartitions) { assertTrueEventually(new AssertTask() { @Override public void run() throws Exception { if (index < numberOfNodesToCrash) { assertLostPartitions(log, listener, survivingPartitions); } else { String message = log + " listener-" + index + " should not be invoked!"; assertTrue(message, listener.getEvents().isEmpty()); } } }); } private void assertLostPartitions(String log, EventCollectingCachePartitionLostListener listener, Map<Integer, Integer> survivingPartitions) { List<CachePartitionLostEvent> events = listener.getEvents(); assertFalse(survivingPartitions.isEmpty()); for (CachePartitionLostEvent event : events) { int failedPartitionId = event.getPartitionId(); Integer survivingReplicaIndex = survivingPartitions.get(failedPartitionId); if (survivingReplicaIndex != null) { String message = log + ", PartitionId: " + failedPartitionId + " SurvivingReplicaIndex: " + survivingReplicaIndex + " Event: " + event.toString(); assertTrue(message, survivingReplicaIndex > listener.getBackupCount()); } } } }