Invocation_RetryTest.java example

Explorer
haze-master
- hazelcast-master
/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.spi.impl.operationservice.impl;

import com.hazelcast.config.Config;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.MemberLeftException;
import com.hazelcast.internal.partition.InternalPartitionService;
import com.hazelcast.nio.Address;
import com.hazelcast.spi.Operation;
import com.hazelcast.spi.OperationService;
import com.hazelcast.spi.PartitionAwareOperation;
import com.hazelcast.spi.exception.RetryableHazelcastException;
import com.hazelcast.spi.impl.AllowedDuringPassiveState;
import com.hazelcast.spi.impl.NodeEngineImpl;
import com.hazelcast.spi.impl.operationservice.InternalOperationService;
import com.hazelcast.spi.partition.IPartition;
import com.hazelcast.spi.properties.GroupProperty;
import com.hazelcast.test.AssertTask;
import com.hazelcast.test.HazelcastParallelClassRunner;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.TestHazelcastInstanceFactory;
import com.hazelcast.test.annotation.ParallelTest;
import com.hazelcast.test.annotation.QuickTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;

@RunWith(HazelcastParallelClassRunner.class)
@Category({QuickTest.class, ParallelTest.class})
public class Invocation_RetryTest extends HazelcastTestSupport {

    private static final int NUMBER_OF_INVOCATIONS = 100;

    @Test
    public void whenPartitionTargetMemberDiesThenOperationSendToNewPartitionOwner() throws Exception {
        TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(2);
        HazelcastInstance local = factory.newHazelcastInstance();
        HazelcastInstance remote = factory.newHazelcastInstance();
        warmUpPartitions(local, remote);

        OperationService service = getOperationService(local);
        Operation op = new PartitionTargetOperation();
        Future future = service.createInvocationBuilder(null, op, getPartitionId(remote))
                .setCallTimeout(30000)
                .invoke();
        sleepSeconds(1);

        remote.shutdown();

        // future.get() should work without a problem because the operation should be re-targeted at the newest owner
        // for the given partition
        future.get();
    }

    @Test(expected = MemberLeftException.class)
    public void whenTargetMemberDiesThenOperationAbortedWithMembersLeftException() throws Exception {
        TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(2);
        HazelcastInstance local = factory.newHazelcastInstance();
        HazelcastInstance remote = factory.newHazelcastInstance();
        warmUpPartitions(local, remote);

        OperationService service = getOperationService(local);
        Operation op = new TargetOperation();
        Address address = new Address(remote.getCluster().getLocalMember().getSocketAddress());
        Future future = service.createInvocationBuilder(null, op, address).invoke();
        sleepSeconds(1);

        remote.getLifecycleService().terminate();

        future.get();
    }

    @Test
    public void testNoStuckInvocationsWhenRetriedMultipleTimes() throws Exception {
        Config config = new Config();
        config.setProperty(GroupProperty.OPERATION_CALL_TIMEOUT_MILLIS.getName(), "3000");

        TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(2);
        HazelcastInstance local = factory.newHazelcastInstance(config);
        HazelcastInstance remote = factory.newHazelcastInstance(config);
        warmUpPartitions(local, remote);

        NodeEngineImpl localNodeEngine = getNodeEngineImpl(local);
        NodeEngineImpl remoteNodeEngine = getNodeEngineImpl(remote);
        final OperationServiceImpl operationService = (OperationServiceImpl) localNodeEngine.getOperationService();

        NonResponsiveOperation op = new NonResponsiveOperation();
        op.setValidateTarget(false);
        op.setPartitionId(1);

        InvocationFuture future = (InvocationFuture) operationService.invokeOnTarget(null, op, remoteNodeEngine.getThisAddress());

        Field invocationField = InvocationFuture.class.getDeclaredField("invocation");
        invocationField.setAccessible(true);
        Invocation invocation = (Invocation) invocationField.get(future);

        invocation.notifyError(new RetryableHazelcastException());
        invocation.notifyError(new RetryableHazelcastException());

        assertTrueEventually(new AssertTask() {
            @Override
            public void run() throws Exception {
                Iterator<Invocation> invocations = operationService.invocationRegistry.iterator();
                assertFalse(invocations.hasNext());
            }
        });
    }

    @Test
    public void invocationShouldComplete_whenRetried_DuringShutdown() throws Exception {
        TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory();
        HazelcastInstance hz1 = factory.newHazelcastInstance();
        HazelcastInstance hz2 = factory.newHazelcastInstance();

        HazelcastInstance hz3 = factory.newHazelcastInstance();
        waitAllForSafeState(hz1, hz2, hz3);

        InternalOperationService operationService = getNodeEngineImpl(hz1).getOperationService();

        Future[] futures = new Future[NUMBER_OF_INVOCATIONS];
        for (int i = 0; i < NUMBER_OF_INVOCATIONS; i++) {
            int partitionId = getRandomPartitionId(hz2);

            futures[i] = operationService.createInvocationBuilder(null, new RetryingOperation(), partitionId)
                    .setTryCount(Integer.MAX_VALUE)
                    .setCallTimeout(Long.MAX_VALUE)
                    .invoke();
        }

        hz3.getLifecycleService().terminate();
        hz1.getLifecycleService().terminate();

        for (Future future : futures) {
            try {
                future.get(2, TimeUnit.MINUTES);
            } catch (ExecutionException ignored) {
            } catch (TimeoutException e) {
                fail(e.getMessage());
            }
        }
    }

    @Test
    public void invocationShouldComplete_whenOperationsPending_DuringShutdown() throws Exception {
        TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory();
        HazelcastInstance hz = factory.newHazelcastInstance();

        NodeEngineImpl nodeEngine = getNodeEngineImpl(hz);
        InternalOperationService operationService = nodeEngine.getOperationService();

        operationService.invokeOnPartition(new SleepingOperation(Long.MAX_VALUE).setPartitionId(0));
        sleepSeconds(1);

        Future[] futures = new Future[NUMBER_OF_INVOCATIONS];
        for (int i = 0; i < NUMBER_OF_INVOCATIONS; i++) {
            futures[i] = operationService.createInvocationBuilder(null, new RetryingOperation(), 0)
                    .setTryCount(Integer.MAX_VALUE)
                    .setCallTimeout(Long.MAX_VALUE)
                    .invoke();
        }

        hz.getLifecycleService().terminate();

        for (Future future : futures) {
            try {
                future.get(2, TimeUnit.MINUTES);
            } catch (ExecutionException ignored) {
            } catch (TimeoutException e) {
                fail(e.getMessage());
            }
        }
    }

    private static int getRandomPartitionId(HazelcastInstance hz) {
        warmUpPartitions(hz);

        InternalPartitionService partitionService = getPartitionService(hz);
        IPartition[] partitions = partitionService.getPartitions();
        Collections.shuffle(Arrays.asList(partitions));

        for (IPartition p : partitions) {
            if (p.isLocal()) {
                return p.getPartitionId();
            }
        }
        throw new RuntimeException("No local partitions are found for hz: " + hz.getName());
    }

    /**
     * Non-responsive operation.
     */
    public static class NonResponsiveOperation extends Operation {

        @Override
        public void run() throws InterruptedException {
        }

        @Override
        public boolean returnsResponse() {
            return false;
        }
    }

    /**
     * Operation send to a specific member.
     */
    private static class TargetOperation extends Operation {

        @Override
        public void run() throws InterruptedException {
            Thread.sleep(10000);
        }
    }

    /**
     * Operation send to a specific target partition.
     */
    private static class PartitionTargetOperation extends Operation implements PartitionAwareOperation {

        @Override
        public void run() throws InterruptedException {
            Thread.sleep(5000);
        }
    }

    private static class RetryingOperation extends Operation implements AllowedDuringPassiveState {

        @Override
        public void run() throws Exception {
            throw new RetryableHazelcastException();
        }
    }

    private static class SleepingOperation extends Operation implements AllowedDuringPassiveState {

        private long sleepMillis;

        public SleepingOperation() {
        }

        SleepingOperation(long sleepMillis) {
            this.sleepMillis = sleepMillis;
        }

        @Override
        public void run() throws Exception {
            Thread.sleep(sleepMillis);
        }
    }
}