/* * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.hazelcast.cluster; import com.hazelcast.cluster.SplitBrainHandlerTest.MergedEventLifeCycleListener; import com.hazelcast.config.Config; import com.hazelcast.core.Hazelcast; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.core.MemberAttributeEvent; import com.hazelcast.core.MembershipEvent; import com.hazelcast.core.MembershipListener; import com.hazelcast.instance.HazelcastInstanceFactory; import com.hazelcast.internal.cluster.impl.MulticastJoiner; import com.hazelcast.spi.properties.GroupProperty; import com.hazelcast.test.AssertTask; import com.hazelcast.test.HazelcastSerialClassRunner; import com.hazelcast.test.annotation.NightlyTest; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @RunWith(HazelcastSerialClassRunner.class) @Category(NightlyTest.class) public class SlowMulticastJoinTest extends AbstractJoinTest { @Before @After public void killAllHazelcastInstances() throws IOException { HazelcastInstanceFactory.terminateAll(); } @Test public void testMembersStaysIndependentWhenHostIsNotTrusted() { Config config1 = newConfig("8.8.8.8"); //8.8.8.8 is never a local address Config config2 = newConfig("8.8.8.8"); int testDurationSeconds = 30; assertIndependentClustersAndDoNotMergedEventually(config1, config2, testDurationSeconds); } @Test public void testMembersFormAClusterWhenHostIsTrusted() throws Exception { Config config2 = newConfig("*.*.*.*"); //matching everything testJoin(config2); } @Test public void testSplitBrainMessagesNotAccumulated_whenClusterIsStableOrNodeIsNotMaster() throws Exception { final int clusterSize = 3; Config config = new Config(); config.setProperty(GroupProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); config.setProperty(GroupProperty.MERGE_FIRST_RUN_DELAY_SECONDS.getName(), "10"); config.setProperty(GroupProperty.MERGE_NEXT_RUN_DELAY_SECONDS.getName(), "15"); final HazelcastInstance[] instances = new HazelcastInstance[clusterSize]; final MulticastJoiner[] joiners = new MulticastJoiner[clusterSize]; for (int i = 0; i < clusterSize; i++) { instances[i] = Hazelcast.newHazelcastInstance(config); joiners[i] = (MulticastJoiner) getNode(instances[i]).getJoiner(); } assertClusterSize(clusterSize, instances[0]); // we will split the cluster to subclusters (0, 1), (2) final CountDownLatch splitLatch = new CountDownLatch(2); instances[2].getCluster().addMembershipListener(new MembershipListener() { @Override public void memberAdded(MembershipEvent membershipEvent) { } @Override public void memberRemoved(MembershipEvent membershipEvent) { splitLatch.countDown(); } @Override public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) { } }); final CountDownLatch mergeLatch = new CountDownLatch(1); instances[2].getLifecycleService().addLifecycleListener(new MergedEventLifeCycleListener(mergeLatch)); // while cluster is stable, split brain join messages should not be accumulated. // the master member for the 10 seconds duration of assertion may have up to as many as other members // in the cluster (clusterSize-1) messages in its queue which will be ignored by the SplitBrainHandler. assertSplitBrainMessagesCount(clusterSize, instances, joiners); // split cluster closeConnectionBetween(instances[0], instances[2]); closeConnectionBetween(instances[1], instances[2]); assertTrue(splitLatch.await(10, TimeUnit.SECONDS)); // while cluster is split, no split brain join messages should be accumulated in the non-master member 1 assertSplitBrainMessagesCount(clusterSize, new HazelcastInstance[] {instances[1]}, new MulticastJoiner[] {joiners[1]}); assertTrue(mergeLatch.await(30, TimeUnit.SECONDS)); assertClusterSize(clusterSize, instances[0]); // cluster is merged & stable again, split brain join messages should not be accumulated. assertSplitBrainMessagesCount(clusterSize, instances, joiners); } private void assertSplitBrainMessagesCount(final int clusterSize, final HazelcastInstance[] instances, final MulticastJoiner[] joiners) { assertTrueAllTheTime(new AssertTask() { @Override public void run() throws Exception { for (int i = 0; i < instances.length; i++) { // a master can have at most (clusterSize-1) split brain join messages if (getNode(instances[i]).isMaster()) { assertTrue(joiners[i].getSplitBrainMessagesCount() < clusterSize); } else { // other members should not have any split brain join messages assertEquals(0, joiners[i].getSplitBrainMessagesCount()); } } } }, 10); } private Config newConfig(String trustedInterface) { Config config = new Config(); config.setProperty(GroupProperty.MERGE_FIRST_RUN_DELAY_SECONDS.getName(), "5"); config.setProperty(GroupProperty.MERGE_NEXT_RUN_DELAY_SECONDS.getName(), "3"); config.getNetworkConfig().getJoin().getMulticastConfig().addTrustedInterface(trustedInterface); return config; } }