package com.leansoft.luxun.quickstart;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.leansoft.luxun.common.exception.ConsumerTimeoutException;
import com.leansoft.luxun.consumer.ConsumerConfig;
import com.leansoft.luxun.consumer.IStreamFactory;
import com.leansoft.luxun.consumer.MessageStream;
import com.leansoft.luxun.consumer.StreamFactory;
import com.leansoft.luxun.producer.IStringProducer;
import com.leansoft.luxun.producer.ProducerConfig;
import com.leansoft.luxun.producer.ProducerData;
import com.leansoft.luxun.serializer.StringDecoder;
import com.leansoft.luxun.server.LuxunServer;
import com.leansoft.luxun.server.ServerConfig;
import com.leansoft.luxun.utils.ImmutableMap;
import com.leansoft.luxun.utils.TestUtils;
import com.leansoft.luxun.utils.Utils;
public class AdvancedDemo {
// enable or disable async producing
private static final boolean async = true;
// 2 topics
private static final String TOPIC_STAR = "star";
private static final String TOPIC_MOON = "moon";
// number of items to produce per topic
private static int topicStarTotalItemCount = 100000;
private static int topicMoonTotalItemCount = 200000;
// number of producer threads
private static int topicStarProducerNum = 2;
private static int topicMoonProducerNum = 4;
private static final AtomicInteger topicStarProducedItemCount = new AtomicInteger(0);
private static final AtomicInteger topicMoonProducedItemCount = new AtomicInteger(0);
// remember items produced for later validation
private static final List<String> topicStarProducedItems = Collections.synchronizedList(new ArrayList<String>());
private static final List<String> topicMoonProducedItems = Collections.synchronizedList(new ArrayList<String>());
// number of consumer threads per topic/group
private static int topicStarGroupAConsumerNum = 2;
private static int topicStarGroupBConsumerNum = 4;
private static int topicMoonGroupAConsumerNum = 1;
private static int topicMoonGroupBConsumerNum = 2;
// remember items consumed for later validation
private static final List<String> topicStarGroupAConsumedItems = Collections.synchronizedList(new ArrayList<String>());
private static final List<String> topicStarGroupBConsumedItems = Collections.synchronizedList(new ArrayList<String>());
private static final List<String> topicMoonGroupAConsumedItems = Collections.synchronizedList(new ArrayList<String>());
private static final List<String> topicMoonGroupBConsumedItems = Collections.synchronizedList(new ArrayList<String>());
private int brokerId1 = 0;
private int brokerId2 = 1;
private int port1 = 9092;
private int port2 = 9093;
private LuxunServer server1 = null;
private LuxunServer server2 = null;
private String brokerList = brokerId1 + ":127.0.0.1:" + port1 + "," + brokerId2 + ":127.0.0.1:" + port2;
@Before
public void setup() {
// set up 2 brokers
Properties props1 = new Properties();
props1.put("brokerid", String.valueOf(brokerId1));
props1.put("port", String.valueOf(port1));
props1.put("log.dir", TestUtils.createTempDir().getAbsolutePath());
ServerConfig config1 = new ServerConfig(props1);
server1 = new LuxunServer(config1);
server1.startup();
Properties props2 = new Properties();
props2.put("brokerid", String.valueOf(brokerId2));
props2.put("port", String.valueOf(port2));
props2.put("log.dir", TestUtils.createTempDir().getAbsolutePath());
ServerConfig config2 = new ServerConfig(props2);
server2 = new LuxunServer(config2);
server2.startup();
}
// Thread produces to topic star
static class StarProducer extends Thread {
private final CountDownLatch latch;
private final IStringProducer stringProducer;
public StarProducer(CountDownLatch latch, IStringProducer stringProducer) {
this.latch = latch;
this.stringProducer = stringProducer;
}
public void run() {
latch.countDown();
try {
latch.await();
} catch (InterruptedException e) {
// ignore
}
while(true) {
int count = topicStarProducedItemCount.incrementAndGet();
if(count > topicStarTotalItemCount) break;
String item = TOPIC_STAR + '-' + count;
topicStarProducedItems.add(item);
stringProducer.send(new ProducerData<String, String>(TOPIC_STAR, item));
}
}
}
// Thread produces to topic moon
static class MoonProducer extends Thread {
private final CountDownLatch latch;
private final IStringProducer stringProducer;
public MoonProducer(CountDownLatch latch, IStringProducer stringProducer) {
this.latch = latch;
this.stringProducer = stringProducer;
}
public void run() {
latch.countDown();
try {
latch.await();
} catch (InterruptedException e) {
// ignore
}
while(true) {
int count = topicMoonProducedItemCount.incrementAndGet();
if(count > topicMoonTotalItemCount) break;
String item = TOPIC_MOON + '-' + count;
topicMoonProducedItems.add(item);
stringProducer.send(new ProducerData<String, String>(TOPIC_MOON, item));
}
}
}
static class Consumer extends Thread {
private final CountDownLatch latch;
private final MessageStream<String> stream;
List<String> consumedItems;
public Consumer(CountDownLatch latch,
MessageStream<String> stream, List<String> consumedItems) {
this.latch = latch;
this.stream = stream;
this.consumedItems = consumedItems;
}
public void run() {
latch.countDown();
try {
latch.await();
} catch (InterruptedException e) {
// ignore
}
try {
for(String item : stream) {
this.consumedItems.add(item);
}
} catch (ConsumerTimeoutException cte) {
// expected
}
}
}
@Test
public void demo() throws IOException {
//prepare
CountDownLatch allLatch = new CountDownLatch(topicStarProducerNum + topicMoonProducerNum +
topicStarGroupAConsumerNum + topicStarGroupBConsumerNum + topicMoonGroupAConsumerNum + topicMoonGroupBConsumerNum);
IStringProducer[] topicStarProducers = new IStringProducer[topicStarProducerNum];
IStringProducer[] topicMoonProducers = new IStringProducer[topicMoonProducerNum];
List<Thread> producers = new ArrayList<Thread>();
List<Thread> consumers = new ArrayList<Thread>();
//start topic star producers
for(int i = 0; i < topicStarProducerNum; i++) {
Properties props = new Properties();
props.put("broker.list", this.brokerList);
if (async) {
props.put("producer.type", "async");
props.put("queue.enqueueTimeout.ms", "-1");
props.put("queue.time", "500");
}
ProducerConfig config = new ProducerConfig(props);
IStringProducer stringProducer = new com.leansoft.luxun.producer.StringProducer(config);
topicStarProducers[i] = stringProducer;
StarProducer p = new StarProducer(allLatch, stringProducer);
producers.add(p);
p.start();
}
//start topic moon producers
for(int i = 0; i < topicMoonProducerNum; i++) {
Properties props = new Properties();
props.put("broker.list", this.brokerList);
if (async) {
props.put("producer.type", "async");
props.put("queue.enqueueTimeout.ms", "-1");
props.put("queue.time", "500");
}
ProducerConfig config = new ProducerConfig(props);
IStringProducer stringProducer = new com.leansoft.luxun.producer.StringProducer(config);
topicMoonProducers[i] = stringProducer;
MoonProducer p = new MoonProducer(allLatch, stringProducer);
producers.add(p);
p.start();
}
// start group A consumers
Properties props = new Properties();
props.put("broker.list", brokerList);
props.put("groupid", "group-a");
props.put("consumerid", "consume-1");
props.put("consumer.timeout.ms", String.valueOf(5000)); // let timeout if no message consumed in 5 seconds.
ConsumerConfig groupAConsumerConfig = new ConsumerConfig(props);
IStreamFactory groupAStreamFactory = new StreamFactory(groupAConsumerConfig);
Map<String, List<MessageStream<String>>> groupAStreams = groupAStreamFactory.createMessageStreams(
ImmutableMap.of(TOPIC_STAR, topicStarGroupAConsumerNum, TOPIC_MOON, topicMoonGroupAConsumerNum), new StringDecoder());
// delegate stream consuming to consumer threads
List<MessageStream<String>> topicFishGroupAStreams = groupAStreams.get(TOPIC_STAR);
for(MessageStream<String> stream : topicFishGroupAStreams) {
Consumer c = new Consumer(allLatch, stream, topicStarGroupAConsumedItems);
consumers.add(c);
c.start();
}
List<MessageStream<String>> topicMoonGroupAStreams = groupAStreams.get(TOPIC_MOON);
for(MessageStream<String> stream : topicMoonGroupAStreams) {
Consumer c = new Consumer(allLatch, stream, topicMoonGroupAConsumedItems);
consumers.add(c);
c.start();
}
// start group B consumers
props.put("groupid", "group-b");
props.put("consumerid", "consume-2");
ConsumerConfig groupBConsumerConfig = new ConsumerConfig(props);
IStreamFactory groupBStreamFactory = new StreamFactory(groupBConsumerConfig);
Map<String, List<MessageStream<String>>> groupBStreams = groupBStreamFactory.createMessageStreams(
ImmutableMap.of(TOPIC_STAR, topicStarGroupBConsumerNum, TOPIC_MOON, topicMoonGroupBConsumerNum), new StringDecoder());
// delegate stream consuming to consumer threads
List<MessageStream<String>> topicStartGroupBStreams = groupBStreams.get(TOPIC_STAR);
for(MessageStream<String> stream : topicStartGroupBStreams) {
Consumer c = new Consumer(allLatch, stream, topicStarGroupBConsumedItems);
consumers.add(c);
c.start();
}
List<MessageStream<String>> topicMoonGroupBStreams = groupBStreams.get(TOPIC_MOON);
for(MessageStream<String> stream : topicMoonGroupBStreams) {
Consumer c = new Consumer(allLatch, stream, topicMoonGroupBConsumedItems);
consumers.add(c);
c.start();
}
// wait threads to finish
for(Thread thread: producers) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
for(Thread thread: consumers) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
// validation
assertTrue(topicStarProducedItems.size() == topicStarTotalItemCount);
assertTrue(topicMoonProducedItems.size() == topicMoonTotalItemCount);
assertTrue(this.assertListEquals(topicStarProducedItems, topicStarGroupAConsumedItems));
assertTrue(this.assertListEquals(topicStarProducedItems, topicStarGroupBConsumedItems));
assertTrue(this.assertListEquals(topicMoonProducedItems, topicMoonGroupAConsumedItems));
assertTrue(this.assertListEquals(topicMoonProducedItems, topicMoonGroupBConsumedItems));
// close
for(int i = 0; i < topicStarProducerNum; i++) {
topicStarProducers[i].close();
}
for(int i = 0; i < topicMoonProducerNum; i++) {
topicMoonProducers[i].close();
}
groupAStreamFactory.close();
groupBStreamFactory.close();
}
boolean assertListEquals(List<String> source, List<String> target) {
if (source.size() != target.size()) return false;
Collections.sort(source);
Collections.sort(target);
for(int i = 0; i < source.size(); i++) {
if (!source.get(i).equals(target.get(i))) {
return false;
}
}
return true;
}
@After
public void cleanup() throws Exception {
server1.close();
server2.close();
Utils.deleteDirectory(new File(server1.config.getLogDir()));
Utils.deleteDirectory(new File(server2.config.getLogDir()));
Thread.sleep(500);
}
}