/* * JBoss, Home of Professional Open Source * Copyright 2010 Red Hat Inc. and/or its affiliates and other * contributors as indicated by the @author tags. All rights reserved. * See the copyright.txt in the distribution for a full listing of * individual contributors. * * This is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this software; if not, write to the Free * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA * 02110-1301 USA, or see the FSF site: http://www.fsf.org. */ package org.infinispan.replication; import org.infinispan.Cache; import org.infinispan.commands.ReplicableCommand; import org.infinispan.config.Configuration; import org.infinispan.config.GlobalConfiguration; import org.infinispan.manager.CacheContainer; import org.infinispan.remoting.ReplicationQueueImpl; import org.infinispan.test.MultipleCacheManagersTest; import org.infinispan.test.TestingUtil; import org.infinispan.test.fwk.TestCacheManagerFactory; import org.infinispan.util.logging.Log; import org.infinispan.util.logging.LogFactory; import org.testng.annotations.Test; import java.lang.reflect.Method; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; /** * Verifies that concurrent flushes are handled properly. These can occur when both flushes due to queue max size * being exceeded and interval based queue flushes occur at exactly the same time. The test verifies that order of * operations is guaranteed under these circumstances. * * @author Galder ZamarreƱo * @since 4.2 */ @Test(groups = "functional", testName = "replication.ConcurrentFlushReplQueueTest") public class ConcurrentFlushReplQueueTest extends MultipleCacheManagersTest { @Override protected void createCacheManagers() throws Throwable { Configuration cfg = new Configuration(); cfg.setCacheMode(Configuration.CacheMode.REPL_ASYNC); cfg.setUseReplQueue(true); cfg.setReplQueueInterval(1000); cfg.setReplQueueMaxElements(2); cfg.setReplQueueClass(MockReplQueue.class.getName()); CacheContainer first = TestCacheManagerFactory.createCacheManager(GlobalConfiguration.getClusteredDefault(), cfg); CacheContainer second = TestCacheManagerFactory.createCacheManager(GlobalConfiguration.getClusteredDefault(), cfg); registerCacheManager(first, second); } public void testConcurrentFlush(Method m) throws Exception { Cache cache1 = cache(0); Cache cache2 = cache(1); CountDownLatch intervalFlushLatch = new CountDownLatch(1); CountDownLatch secondPutLatch = new CountDownLatch(1); CountDownLatch removeCompletedLatch = new CountDownLatch(1); MockReplQueue.intervalFlushLatch = intervalFlushLatch; MockReplQueue.secondPutLatch = secondPutLatch; MockReplQueue.removeCompletedLatch = removeCompletedLatch; final String k = "k-" + m.getName(); final String v = "v-" + m.getName(); cache1.put(k, v); // Wait for periodic repl queue task to try repl the single modification secondPutLatch.await(); // Put something random so that after remove call, the element number exceeds cache1.put("k-blah","v-blah"); cache1.remove(k); // Wait for remove to go over draining the queue removeCompletedLatch.await(1000, TimeUnit.MILLISECONDS); // Once remove executed, now let the interval flush continue intervalFlushLatch.countDown(); // Wait for periodic flush to send modifications over the wire TestingUtil.sleepThread(500); assert !cache2.containsKey(k); } public static class MockReplQueue extends ReplicationQueueImpl { static final Log log = LogFactory.getLog(MockReplQueue.class); static CountDownLatch intervalFlushLatch; static CountDownLatch secondPutLatch; static CountDownLatch removeCompletedLatch; @Override protected List<ReplicableCommand> drainReplQueue() { log.debugf("drainReplQueue called"); List<ReplicableCommand> drained = super.drainReplQueue(); try { if (drained.size() > 0 && Thread.currentThread().getName().startsWith("Scheduled-")) { log.debugf("Drained the put command on the replication thread: %s", drained); secondPutLatch.countDown(); // Wait a max of 5 seconds, because if a remove could have gone through, // it would have done it in that time. If it hasn't and the test passes, // it means that correct synchronization is in place. intervalFlushLatch.await(5, TimeUnit.SECONDS); } else if (drained.size() > 0) { log.debugf("Drained the put+remove commands on the main thread: %s", drained); removeCompletedLatch.countDown(); } } catch (InterruptedException e) { throw new RuntimeException(e); } return drained; } } }