WriteBehindOnBackupsTest.java

/*
 * Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.map.impl.mapstore.writebehind;

import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IMap;
import com.hazelcast.core.Member;
import com.hazelcast.core.Partition;
import com.hazelcast.core.PartitionService;
import com.hazelcast.map.impl.MapService;
import com.hazelcast.map.impl.MapServiceContext;
import com.hazelcast.map.impl.mapstore.MapDataStore;
import com.hazelcast.map.impl.recordstore.RecordStore;
import com.hazelcast.spi.impl.NodeEngineImpl;
import com.hazelcast.spi.properties.GroupProperty;
import com.hazelcast.test.AssertTask;
import com.hazelcast.test.HazelcastParallelClassRunner;
import com.hazelcast.test.HazelcastTestSupport;
import com.hazelcast.test.TestHazelcastInstanceFactory;
import com.hazelcast.test.annotation.ParallelTest;
import com.hazelcast.test.annotation.QuickTest;
import com.hazelcast.test.annotation.SlowTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.util.UUID;
import java.util.concurrent.TimeUnit;

import static org.junit.Assert.assertEquals;

@RunWith(HazelcastParallelClassRunner.class)
@Category({QuickTest.class, ParallelTest.class})
public class WriteBehindOnBackupsTest extends HazelcastTestSupport {

    /**
     * {@link com.hazelcast.map.impl.mapstore.writebehind.StoreWorker} delays processing of write-behind queues (wbq) by adding
     * delay with {@link GroupProperty#MAP_REPLICA_SCHEDULED_TASK_DELAY_SECONDS} property.
     * This is used to provide some extra robustness against node disaster scenarios by trying to prevent lost of entries in wbq-s.
     * Normally backup nodes don't store entries only remove them from wbq-s. Here, we are testing removal of entries occurred or not.
     */
    @Test
    public void testBackupRemovesEntries_afterProcessingDelay() throws Exception {
        final int numberOfItems = 10;
        final String mapName = randomMapName();
        final MapStoreWithCounter<Integer, Integer> mapStore = new MapStoreWithCounter<Integer, Integer>();
        TestMapUsingMapStoreBuilder<Integer, Integer> storeBuilder = TestMapUsingMapStoreBuilder.create();
        final IMap<Integer, Integer> map = storeBuilder
                .mapName(mapName)
                .withMapStore(mapStore)
                .withNodeCount(2)
                .withNodeFactory(createHazelcastInstanceFactory(2))
                .withWriteDelaySeconds(1)
                .withBackupCount(1)
                .withPartitionCount(1)
                .withBackupProcessingDelay(1)
                .build();

        populateMap(map, numberOfItems);

        assertWriteBehindQueuesEmptyOnOwnerAndOnBackups(mapName, numberOfItems, mapStore, storeBuilder.getNodes());
    }

    @Test
    public void testPutTransientDoesNotStoreEntry_onBackupPartition() {
        String mapName = randomMapName();
        final MapStoreWithCounter<Integer, Integer> mapStore = new MapStoreWithCounter<Integer, Integer>();
        TestMapUsingMapStoreBuilder<Integer, Integer> storeBuilder = TestMapUsingMapStoreBuilder.create();
        final IMap<Integer, Integer> map = storeBuilder
                .mapName(mapName)
                .withMapStore(mapStore)
                .withNodeCount(2)
                .withNodeFactory(createHazelcastInstanceFactory(2))
                .withWriteDelaySeconds(1)
                .withBackupCount(1)
                .withPartitionCount(1)
                .withBackupProcessingDelay(1)
                .build();

        map.putTransient(1, 1, 1, TimeUnit.DAYS);

        sleepSeconds(5);

        assertEquals("There should not be any store operation", 0, mapStore.countStore.get());
    }

    @Test
    @Category(SlowTest.class)
    public void testPutTransientDoesNotStoreEntry_onPromotedReplica() {
        String mapName = randomMapName();
        final MapStoreWithCounter<String, Object> mapStore = new MapStoreWithCounter<String, Object>();
        TestMapUsingMapStoreBuilder<String, Object> storeBuilder = TestMapUsingMapStoreBuilder.create();
        TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(2);
        final IMap<String, Object> map = storeBuilder
                .mapName(mapName)
                .withMapStore(mapStore)
                .withNodeCount(2)
                .withNodeFactory(factory)
                .withWriteDelaySeconds(5)
                .withBackupCount(1)
                .withPartitionCount(1)
                .withBackupProcessingDelay(1)
                .build();

        String key = UUID.randomUUID().toString();

        map.putTransient(key, 1, 1, TimeUnit.DAYS);

        killKeyOwner(key, storeBuilder);

        sleepSeconds(10);

        assertEquals("There should not be any store operation on promoted replica", 0, mapStore.countStore.get());
    }

    private void killKeyOwner(String key, TestMapUsingMapStoreBuilder<String, Object> storeBuilder) {
        HazelcastInstance[] nodes = storeBuilder.getNodes();
        HazelcastInstance ownerNode = getOwnerNode(key, nodes);
        ownerNode.shutdown();
    }

    private HazelcastInstance getOwnerNode(String key, HazelcastInstance[] nodes) {
        PartitionService partitionService = nodes[0].getPartitionService();
        Partition partition = partitionService.getPartition(key);
        Member owner = partition.getOwner();

        for (HazelcastInstance node : nodes) {
            Member localMember = node.getCluster().getLocalMember();
            if (localMember.equals(owner)) {
                return node;
            }
        }

        throw new IllegalStateException("This should not be happen...");
    }

    private void assertWriteBehindQueuesEmptyOnOwnerAndOnBackups(final String mapName,
                                                                 final long numberOfItems,
                                                                 final MapStoreWithCounter mapStore,
                                                                 final HazelcastInstance[] nodes) {
        AssertTask assertTask = new AssertTask() {
            @Override
            public void run() throws Exception {
                assertEquals(0, writeBehindQueueSize(nodes[0], mapName));
                assertEquals(0, writeBehindQueueSize(nodes[1], mapName));

                assertEquals(numberOfItems, mapStore.size());
            }
        };

        assertTrueEventually(assertTask);
    }

    private void populateMap(IMap<Integer, Integer> map, int numberOfItems) {
        for (int i = 0; i < numberOfItems; i++) {
            map.put(i, i);
        }
    }

    public static int writeBehindQueueSize(HazelcastInstance node, String mapName) {
        int size = 0;
        final NodeEngineImpl nodeEngine = getNode(node).getNodeEngine();
        MapService mapService = nodeEngine.getService(MapService.SERVICE_NAME);
        final MapServiceContext mapServiceContext = mapService.getMapServiceContext();
        final int partitionCount = nodeEngine.getPartitionService().getPartitionCount();
        for (int i = 0; i < partitionCount; i++) {
            final RecordStore recordStore = mapServiceContext.getExistingRecordStore(i, mapName);
            if (recordStore == null) {
                continue;
            }
            final MapDataStore mapDataStore = recordStore.getMapDataStore();
            if (mapDataStore instanceof WriteBehindStore) {
                size += ((WriteBehindStore) mapDataStore).getWriteBehindQueue().size();
            }
        }
        return size;
    }
}