package com.leansoft.luxun.load; import static org.junit.Assert.*; import java.io.File; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import org.junit.After; import org.junit.Before; import org.junit.Test; import com.leansoft.luxun.common.exception.TopicNotExistException; import com.leansoft.luxun.consumer.SimpleConsumer; import com.leansoft.luxun.message.Message; import com.leansoft.luxun.message.MessageList; import com.leansoft.luxun.producer.IProducer; import com.leansoft.luxun.producer.ProducerConfig; import com.leansoft.luxun.producer.ProducerData; import com.leansoft.luxun.serializer.Decoder; import com.leansoft.luxun.serializer.StringDecoder; import com.leansoft.luxun.serializer.StringEncoder; import com.leansoft.luxun.server.LuxunServer; import com.leansoft.luxun.server.ServerConfig; import com.leansoft.luxun.utils.TestUtils; import com.leansoft.luxun.utils.Utils; public class SimpleConsumeByFanoutIdLoadTest { private int port = 9092; private int brokerId = 0; private LuxunServer server = null; private String brokerList = brokerId + ":127.0.0.1:" + port; @Before public void setup() { Properties props1 = TestUtils.createBrokerConfig(brokerId, port); props1.setProperty("log.flush.count", "1000"); ServerConfig config1 = new ServerConfig(props1); server = TestUtils.createServer(config1); } @After public void clean() throws Exception { server.close(); Utils.deleteDirectory(new File(server.config.getLogDir())); Thread.sleep(500); } // configurable parameters ////////////////////////////////////////////////////////////////// private static int loop = 5; private static int totalItemCount = 100000; private static int producerNum = 4; private static int consumerGroupANum = 2; private static int consumerGroupBNum = 4; private static int messageLength = 1024; ////////////////////////////////////////////////////////////////// private static enum Status { ERROR, SUCCESS } private static class Result { Status status; } private static final AtomicInteger producingItemCount = new AtomicInteger(0); private static final Map<String, AtomicInteger> itemMap = new ConcurrentHashMap<String,AtomicInteger>(); private static class ProducerThread extends Thread { private final CountDownLatch latch; private final Queue<Result> resultQueue; private final IProducer<String, String> stringProducer; private final String topic; public ProducerThread(CountDownLatch latch, Queue<Result> resultQueue, IProducer<String, String> stringProducer, String topic) { this.latch = latch; this.resultQueue = resultQueue; this.stringProducer = stringProducer; this.topic = topic; } public void run() { Result result = new Result(); String rndString = TestUtils.randomString(messageLength); try { latch.countDown(); latch.await(); while(true) { int count = producingItemCount.incrementAndGet(); if(count > totalItemCount) break; String item = rndString + '-' + count; itemMap.put(item, new AtomicInteger(0)); stringProducer.send(new ProducerData<String, String>(topic, item)); } result.status = Status.SUCCESS; } catch (Exception e) { e.printStackTrace(); result.status = Status.ERROR; } resultQueue.offer(result); } } // sequential consumer can work concurrently with producer private static class SequentialConsumerThread extends Thread { private final CountDownLatch latch; private final Queue<Result> resultQueue; private final SimpleConsumer simpleConsumer; private final String topic; private final Decoder<String> stringDecoder = new StringDecoder(); private final String fanoutId; private final AtomicInteger itemCount; public SequentialConsumerThread(CountDownLatch latch, Queue<Result> resultQueue, String fanoutId, SimpleConsumer simpleConsumer, String topic, AtomicInteger itemCount) { this.latch = latch; this.resultQueue = resultQueue; this.simpleConsumer = simpleConsumer; this.topic = topic; this.fanoutId = fanoutId; this.itemCount = itemCount; } public void run() { Result result = new Result(); try { latch.countDown(); latch.await(); while (itemCount.get() < totalItemCount) { try { List<MessageList> listOfMessageList = simpleConsumer.consume(topic, fanoutId, 10000); if (listOfMessageList.size() == 0) { Thread.sleep(20); // no item to consume yet, just wait a moment } for(MessageList messageList : listOfMessageList) { for(Message message : messageList) { String item = stringDecoder.toEvent(message); AtomicInteger counter = itemMap.get(item); assertNotNull(counter); counter.incrementAndGet(); itemCount.incrementAndGet(); } } } catch (TopicNotExistException ex) { Thread.sleep(200);// wait the producer to register the topic in broker } } result.status = Status.SUCCESS; } catch (Exception e) { e.printStackTrace(); result.status = Status.ERROR; } resultQueue.offer(result); } } public void doRunMixed(int round) throws Exception { //prepare CountDownLatch allLatch = new CountDownLatch(producerNum + consumerGroupANum + consumerGroupBNum); @SuppressWarnings("unchecked") IProducer<String, String>[] producers = new IProducer[producerNum]; SimpleConsumer[] groupAConsumers = new SimpleConsumer[consumerGroupANum]; SimpleConsumer[] groupBConsumers = new SimpleConsumer[consumerGroupBNum]; BlockingQueue<Result> producerResults = new LinkedBlockingQueue<Result>(); BlockingQueue<Result> consumerResults = new LinkedBlockingQueue<Result>(); String topic = "load-test002-" + round; //run testing for(int i = 0; i < producerNum; i++) { Properties props = new Properties(); props.put("serializer.class", StringEncoder.class.getName()); props.put("broker.list", this.brokerList); ProducerConfig config = new ProducerConfig(props); IProducer<String, String> stringProducer = new com.leansoft.luxun.producer.Producer<String, String>(config); producers[i] = stringProducer; ProducerThread p = new ProducerThread(allLatch, producerResults, stringProducer, topic); p.start(); } AtomicInteger groupAItemCount = new AtomicInteger(0); for(int i = 0; i < consumerGroupANum; i++) { SimpleConsumer simpleConsumer = new SimpleConsumer("127.0.0.1", 9092, 60000); groupAConsumers[i] = simpleConsumer; SequentialConsumerThread c = new SequentialConsumerThread(allLatch, consumerResults, "group-a", simpleConsumer, topic, groupAItemCount); c.start(); } AtomicInteger groupBItemCount = new AtomicInteger(0); for(int i = 0; i < consumerGroupBNum; i++) { SimpleConsumer simpleConsumer = new SimpleConsumer("127.0.0.1", 9092, 60000); groupBConsumers[i] = simpleConsumer; SequentialConsumerThread c = new SequentialConsumerThread(allLatch, consumerResults, "group-b", simpleConsumer, topic, groupBItemCount); c.start(); } //verify for(int i = 0; i < producerNum; i++) { Result result = producerResults.take(); assertEquals(result.status, Status.SUCCESS); } for(int i = 0; i < consumerGroupANum + consumerGroupBNum; i++) { Result result = consumerResults.take(); assertEquals(result.status, Status.SUCCESS); } assertTrue(itemMap.size() == totalItemCount); for(AtomicInteger counter : itemMap.values()) { assertTrue(counter.get() == 2); } // closing for(int i = 0; i < producerNum; i++) { producers[i].close(); } for(int i = 0; i < consumerGroupANum; i++) { groupAConsumers[i].close(); } for(int i = 0; i < consumerGroupBNum; i++) { groupBConsumers[i].close(); } } @Test public void runTest() throws Exception { System.out.println("Load test begin ..."); for(int i = 0; i < loop; i++) { System.out.println("[doRunMixed] round " + (i + 1) + " of " + loop); this.doRunMixed(i); // reset producingItemCount.set(0); itemMap.clear(); } System.out.println("Load test finished successfully."); } }