package com.leansoft.luxun.quickstart; import static org.junit.Assert.*; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import org.junit.After; import org.junit.Before; import org.junit.Test; import com.leansoft.luxun.common.exception.ConsumerTimeoutException; import com.leansoft.luxun.consumer.ConsumerConfig; import com.leansoft.luxun.consumer.IStreamFactory; import com.leansoft.luxun.consumer.MessageStream; import com.leansoft.luxun.consumer.StreamFactory; import com.leansoft.luxun.producer.IStringProducer; import com.leansoft.luxun.producer.ProducerConfig; import com.leansoft.luxun.producer.ProducerData; import com.leansoft.luxun.serializer.StringDecoder; import com.leansoft.luxun.server.LuxunServer; import com.leansoft.luxun.server.ServerConfig; import com.leansoft.luxun.utils.ImmutableMap; import com.leansoft.luxun.utils.TestUtils; import com.leansoft.luxun.utils.Utils; public class AdvancedDemo { // enable or disable async producing private static final boolean async = true; // 2 topics private static final String TOPIC_STAR = "star"; private static final String TOPIC_MOON = "moon"; // number of items to produce per topic private static int topicStarTotalItemCount = 100000; private static int topicMoonTotalItemCount = 200000; // number of producer threads private static int topicStarProducerNum = 2; private static int topicMoonProducerNum = 4; private static final AtomicInteger topicStarProducedItemCount = new AtomicInteger(0); private static final AtomicInteger topicMoonProducedItemCount = new AtomicInteger(0); // remember items produced for later validation private static final List<String> topicStarProducedItems = Collections.synchronizedList(new ArrayList<String>()); private static final List<String> topicMoonProducedItems = Collections.synchronizedList(new ArrayList<String>()); // number of consumer threads per topic/group private static int topicStarGroupAConsumerNum = 2; private static int topicStarGroupBConsumerNum = 4; private static int topicMoonGroupAConsumerNum = 1; private static int topicMoonGroupBConsumerNum = 2; // remember items consumed for later validation private static final List<String> topicStarGroupAConsumedItems = Collections.synchronizedList(new ArrayList<String>()); private static final List<String> topicStarGroupBConsumedItems = Collections.synchronizedList(new ArrayList<String>()); private static final List<String> topicMoonGroupAConsumedItems = Collections.synchronizedList(new ArrayList<String>()); private static final List<String> topicMoonGroupBConsumedItems = Collections.synchronizedList(new ArrayList<String>()); private int brokerId1 = 0; private int brokerId2 = 1; private int port1 = 9092; private int port2 = 9093; private LuxunServer server1 = null; private LuxunServer server2 = null; private String brokerList = brokerId1 + ":127.0.0.1:" + port1 + "," + brokerId2 + ":127.0.0.1:" + port2; @Before public void setup() { // set up 2 brokers Properties props1 = new Properties(); props1.put("brokerid", String.valueOf(brokerId1)); props1.put("port", String.valueOf(port1)); props1.put("log.dir", TestUtils.createTempDir().getAbsolutePath()); ServerConfig config1 = new ServerConfig(props1); server1 = new LuxunServer(config1); server1.startup(); Properties props2 = new Properties(); props2.put("brokerid", String.valueOf(brokerId2)); props2.put("port", String.valueOf(port2)); props2.put("log.dir", TestUtils.createTempDir().getAbsolutePath()); ServerConfig config2 = new ServerConfig(props2); server2 = new LuxunServer(config2); server2.startup(); } // Thread produces to topic star static class StarProducer extends Thread { private final CountDownLatch latch; private final IStringProducer stringProducer; public StarProducer(CountDownLatch latch, IStringProducer stringProducer) { this.latch = latch; this.stringProducer = stringProducer; } public void run() { latch.countDown(); try { latch.await(); } catch (InterruptedException e) { // ignore } while(true) { int count = topicStarProducedItemCount.incrementAndGet(); if(count > topicStarTotalItemCount) break; String item = TOPIC_STAR + '-' + count; topicStarProducedItems.add(item); stringProducer.send(new ProducerData<String, String>(TOPIC_STAR, item)); } } } // Thread produces to topic moon static class MoonProducer extends Thread { private final CountDownLatch latch; private final IStringProducer stringProducer; public MoonProducer(CountDownLatch latch, IStringProducer stringProducer) { this.latch = latch; this.stringProducer = stringProducer; } public void run() { latch.countDown(); try { latch.await(); } catch (InterruptedException e) { // ignore } while(true) { int count = topicMoonProducedItemCount.incrementAndGet(); if(count > topicMoonTotalItemCount) break; String item = TOPIC_MOON + '-' + count; topicMoonProducedItems.add(item); stringProducer.send(new ProducerData<String, String>(TOPIC_MOON, item)); } } } static class Consumer extends Thread { private final CountDownLatch latch; private final MessageStream<String> stream; List<String> consumedItems; public Consumer(CountDownLatch latch, MessageStream<String> stream, List<String> consumedItems) { this.latch = latch; this.stream = stream; this.consumedItems = consumedItems; } public void run() { latch.countDown(); try { latch.await(); } catch (InterruptedException e) { // ignore } try { for(String item : stream) { this.consumedItems.add(item); } } catch (ConsumerTimeoutException cte) { // expected } } } @Test public void demo() throws IOException { //prepare CountDownLatch allLatch = new CountDownLatch(topicStarProducerNum + topicMoonProducerNum + topicStarGroupAConsumerNum + topicStarGroupBConsumerNum + topicMoonGroupAConsumerNum + topicMoonGroupBConsumerNum); IStringProducer[] topicStarProducers = new IStringProducer[topicStarProducerNum]; IStringProducer[] topicMoonProducers = new IStringProducer[topicMoonProducerNum]; List<Thread> producers = new ArrayList<Thread>(); List<Thread> consumers = new ArrayList<Thread>(); //start topic star producers for(int i = 0; i < topicStarProducerNum; i++) { Properties props = new Properties(); props.put("broker.list", this.brokerList); if (async) { props.put("producer.type", "async"); props.put("queue.enqueueTimeout.ms", "-1"); props.put("queue.time", "500"); } ProducerConfig config = new ProducerConfig(props); IStringProducer stringProducer = new com.leansoft.luxun.producer.StringProducer(config); topicStarProducers[i] = stringProducer; StarProducer p = new StarProducer(allLatch, stringProducer); producers.add(p); p.start(); } //start topic moon producers for(int i = 0; i < topicMoonProducerNum; i++) { Properties props = new Properties(); props.put("broker.list", this.brokerList); if (async) { props.put("producer.type", "async"); props.put("queue.enqueueTimeout.ms", "-1"); props.put("queue.time", "500"); } ProducerConfig config = new ProducerConfig(props); IStringProducer stringProducer = new com.leansoft.luxun.producer.StringProducer(config); topicMoonProducers[i] = stringProducer; MoonProducer p = new MoonProducer(allLatch, stringProducer); producers.add(p); p.start(); } // start group A consumers Properties props = new Properties(); props.put("broker.list", brokerList); props.put("groupid", "group-a"); props.put("consumerid", "consume-1"); props.put("consumer.timeout.ms", String.valueOf(5000)); // let timeout if no message consumed in 5 seconds. ConsumerConfig groupAConsumerConfig = new ConsumerConfig(props); IStreamFactory groupAStreamFactory = new StreamFactory(groupAConsumerConfig); Map<String, List<MessageStream<String>>> groupAStreams = groupAStreamFactory.createMessageStreams( ImmutableMap.of(TOPIC_STAR, topicStarGroupAConsumerNum, TOPIC_MOON, topicMoonGroupAConsumerNum), new StringDecoder()); // delegate stream consuming to consumer threads List<MessageStream<String>> topicFishGroupAStreams = groupAStreams.get(TOPIC_STAR); for(MessageStream<String> stream : topicFishGroupAStreams) { Consumer c = new Consumer(allLatch, stream, topicStarGroupAConsumedItems); consumers.add(c); c.start(); } List<MessageStream<String>> topicMoonGroupAStreams = groupAStreams.get(TOPIC_MOON); for(MessageStream<String> stream : topicMoonGroupAStreams) { Consumer c = new Consumer(allLatch, stream, topicMoonGroupAConsumedItems); consumers.add(c); c.start(); } // start group B consumers props.put("groupid", "group-b"); props.put("consumerid", "consume-2"); ConsumerConfig groupBConsumerConfig = new ConsumerConfig(props); IStreamFactory groupBStreamFactory = new StreamFactory(groupBConsumerConfig); Map<String, List<MessageStream<String>>> groupBStreams = groupBStreamFactory.createMessageStreams( ImmutableMap.of(TOPIC_STAR, topicStarGroupBConsumerNum, TOPIC_MOON, topicMoonGroupBConsumerNum), new StringDecoder()); // delegate stream consuming to consumer threads List<MessageStream<String>> topicStartGroupBStreams = groupBStreams.get(TOPIC_STAR); for(MessageStream<String> stream : topicStartGroupBStreams) { Consumer c = new Consumer(allLatch, stream, topicStarGroupBConsumedItems); consumers.add(c); c.start(); } List<MessageStream<String>> topicMoonGroupBStreams = groupBStreams.get(TOPIC_MOON); for(MessageStream<String> stream : topicMoonGroupBStreams) { Consumer c = new Consumer(allLatch, stream, topicMoonGroupBConsumedItems); consumers.add(c); c.start(); } // wait threads to finish for(Thread thread: producers) { try { thread.join(); } catch (InterruptedException e) { // ignore } } for(Thread thread: consumers) { try { thread.join(); } catch (InterruptedException e) { // ignore } } // validation assertTrue(topicStarProducedItems.size() == topicStarTotalItemCount); assertTrue(topicMoonProducedItems.size() == topicMoonTotalItemCount); assertTrue(this.assertListEquals(topicStarProducedItems, topicStarGroupAConsumedItems)); assertTrue(this.assertListEquals(topicStarProducedItems, topicStarGroupBConsumedItems)); assertTrue(this.assertListEquals(topicMoonProducedItems, topicMoonGroupAConsumedItems)); assertTrue(this.assertListEquals(topicMoonProducedItems, topicMoonGroupBConsumedItems)); // close for(int i = 0; i < topicStarProducerNum; i++) { topicStarProducers[i].close(); } for(int i = 0; i < topicMoonProducerNum; i++) { topicMoonProducers[i].close(); } groupAStreamFactory.close(); groupBStreamFactory.close(); } boolean assertListEquals(List<String> source, List<String> target) { if (source.size() != target.size()) return false; Collections.sort(source); Collections.sort(target); for(int i = 0; i < source.size(); i++) { if (!source.get(i).equals(target.get(i))) { return false; } } return true; } @After public void cleanup() throws Exception { server1.close(); server2.close(); Utils.deleteDirectory(new File(server1.config.getLogDir())); Utils.deleteDirectory(new File(server2.config.getLogDir())); Thread.sleep(500); } }