package com.leansoft.luxun.integration;
import java.io.File;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.thrift.TException;
import org.junit.Test;
import com.leansoft.luxun.api.generated.ConsumeRequest;
import com.leansoft.luxun.api.generated.ConsumeResponse;
import com.leansoft.luxun.api.generated.ErrorCode;
import com.leansoft.luxun.api.generated.ProduceRequest;
import com.leansoft.luxun.api.generated.ResultCode;
import com.leansoft.luxun.message.Message;
import com.leansoft.luxun.message.MessageList;
import com.leansoft.luxun.message.generated.CompressionCodec;
import com.leansoft.luxun.server.LuxunServer;
import com.leansoft.luxun.server.ServerConfig;
import com.leansoft.luxun.utils.TestUtils;
import com.leansoft.luxun.utils.Utils;
public class LazyInitProducerTest extends ProducerConsumerTestHarness {
Properties props;
ServerConfig config;
List<ServerConfig> configs;
List<LuxunServer> servers;
@Override
public void setUp() throws Exception {
port = TestUtils.choosePort();
props = TestUtils.createBrokerConfig(0, port);
config = new ServerConfig(props);
configs = new ArrayList<ServerConfig>();
configs.add(config);
servers = new ArrayList<LuxunServer>();
servers.add(TestUtils.createServer(config));
super.setUp();
}
@Override
public void tearDown() throws Exception {
super.tearDown();
for(LuxunServer server : servers) {
server.close();
}
for(ServerConfig config : configs) {
Utils.deleteDirectory(new File(config.getLogDir()));
}
}
@Test
public void testProduceAndFetchByIndex() throws TException {
// send some messages
String topic = "test";
MessageList messageList = new MessageList();
messageList.add(new Message("hello".getBytes()));
messageList.add(new Message("there".getBytes()));
producer.send(topic, messageList);
List<MessageList> listOfMessageList = consumer.consume(topic, 0, 10000);
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messageList, listOfMessageList.get(0)));
try {
consumer.consume(topic, -1, 10000);
fail("excepted IndexOutOfBoundsException was not thrown");
} catch(IndexOutOfBoundsException e) {
// expected
}
}
@Test
public void testProduceAndFetchByFanoutId() throws TException {
// send some messages
String topic = "test";
MessageList messageList = new MessageList();
messageList.add(new Message("hello".getBytes()));
messageList.add(new Message("there".getBytes()));
producer.send(topic, messageList);
List<MessageList> listOfMessageList = consumer.consume(topic, "fan1", 10000);
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messageList, listOfMessageList.get(0)));
// fan1 is empty now
listOfMessageList = consumer.consume(topic, "fan1", 10000);
assertTrue(listOfMessageList.isEmpty());
}
@Test
public void testProduceAndFetchOneByOneByIndex() throws TException {
// send some messages
String topic = "test";
for(int i = 0; i < 10; i++) {
MessageList messageList = new MessageList();
messageList.add(new Message(("hello" + i).getBytes()));
producer.send(topic, messageList);
}
// consume one by one
for(int i = 0; i < 5; i++) {
List<MessageList> listOfMessageList = consumer.consume(topic, i, -1);
assertTrue(listOfMessageList.size() == 1);
MessageList msgList = listOfMessageList.get(0);
assertTrue(msgList.size() == 1);
Message msg = msgList.get(0);
assertEquals("hello" + i, new String(msg.getBytes()));
}
// consume remaining in a batch
List<MessageList> listOfMessageList = consumer.consume(topic, 5, 10000);
assertTrue(listOfMessageList.size() == 5);
for(int i = 5; i < 10; i++) {
MessageList msgList = listOfMessageList.get(i - 5);
assertTrue(msgList.size() == 1);
Message msg = msgList.get(0);
assertEquals("hello" + i, new String(msg.getBytes()));
}
}
@Test
public void testProduceAndFetchOneByOneByFanoutId() throws TException {
// send some messages
String topic = "test";
for(int i = 0; i < 10; i++) {
MessageList messageList = new MessageList();
messageList.add(new Message(("hello" + i).getBytes()));
producer.send(topic, messageList);
}
// consume one by one
for(int i = 0; i < 5; i++) {
List<MessageList> listOfMessageList = consumer.consume(topic, "fan", 0);
assertTrue(listOfMessageList.size() == 1);
MessageList msgList = listOfMessageList.get(0);
assertTrue(msgList.size() == 1);
Message msg = msgList.get(0);
assertEquals("hello" + i, new String(msg.getBytes()));
}
// consume remaining in a batch
List<MessageList> listOfMessageList = consumer.consume(topic, "fan", 10000);
assertTrue(listOfMessageList.size() == 5);
for(int i = 5; i < 10; i++) {
MessageList msgList = listOfMessageList.get(i - 5);
assertTrue(msgList.size() == 1);
Message msg = msgList.get(0);
assertEquals("hello" + i, new String(msg.getBytes()));
}
}
public void testProduceAndMultiFetchByIndex() throws TException {
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
for(String topic : topics) {
MessageList messageList = new MessageList();
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
producer.send(topic, messageList);
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setStartIndex(0);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
fetches.clear();
// send some invalid indexes
for(String topic : topics) {
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setStartIndex(-1);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
responses = consumer.multiConsume(fetches);
for(ConsumeResponse response : responses) {
assertTrue(response.getResult().getResultCode() == ResultCode.FAILURE);
assertTrue(response.getResult().getErrorCode() == ErrorCode.INDEX_OUT_OF_BOUNDS);
}
}
public void testProduceAndMultiFetchByFanoutId() throws TException {
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
for(String topic : topics) {
MessageList messageList = new MessageList();
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
producer.send(topic, messageList);
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setFanoutId("fan1");
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
}
public void testProduceAndMultiFetchByIndexWithCompression() throws TException {
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
for(String topic : topics) {
MessageList messageList = new MessageList(CompressionCodec.GZIP);
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
producer.send(topic, messageList);
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setStartIndex(0);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
}
public void testProduceAndMultiFetchByFanoutIdWithCompression() throws TException {
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
for(String topic : topics) {
MessageList messageList = new MessageList(CompressionCodec.GZIP);
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
producer.send(topic, messageList);
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setFanoutId("fan0");
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
}
public void testMultiProduceThenFetchByIndex() throws TException {
// send some messages
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
List<ProduceRequest> produceList = new ArrayList<ProduceRequest>();
for(String topic : topics) {
MessageList messageList = new MessageList();
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
produceList.add(new ProduceRequest(messageList.toThriftBuffer(), topic));
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setStartIndex(0);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
producer.multiSend(produceList);
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
}
public void testMultiProduceThenFetchByFanoutId() throws TException {
// send some messages
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
List<ProduceRequest> produceList = new ArrayList<ProduceRequest>();
int i = 0;
for(String topic : topics) {
MessageList messageList = new MessageList();
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
produceList.add(new ProduceRequest(messageList.toThriftBuffer(), topic));
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setFanoutId("fan" + i);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
i++;
}
producer.multiSend(produceList);
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
}
public void testMultiProduceWithCompressionThenFetchByIndex() throws TException {
// send some messages
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
List<ProduceRequest> produceList = new ArrayList<ProduceRequest>();
for(String topic : topics) {
MessageList messageList = new MessageList(CompressionCodec.GZIP);
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
produceList.add(new ProduceRequest(messageList.toThriftBuffer(), topic));
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setStartIndex(0);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
producer.multiSend(produceList);
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
}
public void testMultiProduceWithCompressionThenFetchByFanoutId() throws TException {
// send some messages
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
List<ProduceRequest> produceList = new ArrayList<ProduceRequest>();
int i = 0;
for(String topic : topics) {
MessageList messageList = new MessageList(CompressionCodec.GZIP);
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
produceList.add(new ProduceRequest(messageList.toThriftBuffer(), topic));
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setFanoutId("fan" + i);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
i++;
}
producer.multiSend(produceList);
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 1);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
index++;
}
}
public void testMultiProduceResendThenFetchByIndex() throws TException {
// send some messages
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
List<ProduceRequest> produceList = new ArrayList<ProduceRequest>();
for(String topic : topics) {
MessageList messageList = new MessageList();
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
produceList.add(new ProduceRequest(messageList.toThriftBuffer(), topic));
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setStartIndex(0);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
}
producer.multiSend(produceList);
// resend the same ultisend
producer.multiSend(produceList);
TestUtils.sleepQuietly(1000); // give time to broker to save the messages
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 2);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(1)));
index++;
}
}
public void testMultiProduceResendThenFetchByFanoutId() throws TException {
// send some messages
Map<String, MessageList> messages = new HashMap<String, MessageList>();
List<String> topics = new ArrayList<String>();
topics.add("test1");
topics.add("test2");
topics.add("test3");
List<ConsumeRequest> fetches = new ArrayList<ConsumeRequest>();
List<ProduceRequest> produceList = new ArrayList<ProduceRequest>();
int i = 0;
for(String topic : topics) {
MessageList messageList = new MessageList();
messageList.add(new Message(("a_" + topic).getBytes()));
messageList.add(new Message(("b_" + topic).getBytes()));
messages.put(topic, messageList);
produceList.add(new ProduceRequest(messageList.toThriftBuffer(), topic));
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(topic);
consumeRequest.setFanoutId("fan" + i);
consumeRequest.setMaxFetchSize(10000);
fetches.add(consumeRequest);
i++;
}
producer.multiSend(produceList);
// resend the same ultisend
producer.multiSend(produceList);
TestUtils.sleepQuietly(1000); // give time to broker to save the messages
List<ConsumeResponse> responses = consumer.multiConsume(fetches);
int index = 0;
for(String topic : topics) {
ConsumeResponse response = responses.get(index);
List<ByteBuffer> itemList = response.getItemList();
List<MessageList> listOfMessageList = new ArrayList<MessageList>();
for(ByteBuffer buffer : itemList) {
listOfMessageList.add(MessageList.fromThriftBuffer(buffer));
}
assertTrue(listOfMessageList.size() == 2);
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(0)));
assertTrue(isMessageListEqual(messages.get(topic), listOfMessageList.get(1)));
index++;
}
}
public void testConsumeNotExistTopicByIndex() throws TException {
String newTopic = "new-topic";
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(newTopic);
consumeRequest.setStartIndex(0);
consumeRequest.setMaxFetchSize(10000);
ConsumeResponse consumeResponse = consumer.consume(consumeRequest);
assertTrue(consumeResponse.getResult().getResultCode() == ResultCode.FAILURE);
assertTrue(consumeResponse.getResult().getErrorCode() == ErrorCode.TOPIC_NOT_EXIST);
File logFile = new File(config.getLogDir(), newTopic);
assertTrue(!logFile.exists());
}
public void testConsumeNotExistTopicByFanoutId() throws TException {
String newTopic = "new-topic";
ConsumeRequest consumeRequest = new ConsumeRequest();
consumeRequest.setTopic(newTopic);
consumeRequest.setFanoutId("fan8");
consumeRequest.setMaxFetchSize(10000);
ConsumeResponse consumeResponse = consumer.consume(consumeRequest);
assertTrue(consumeResponse.getResult().getResultCode() == ResultCode.FAILURE);
assertTrue(consumeResponse.getResult().getErrorCode() == ErrorCode.TOPIC_NOT_EXIST);
File logFile = new File(config.getLogDir(), newTopic);
assertTrue(!logFile.exists());
}
private boolean isMessageListEqual(MessageList source, MessageList target) {
if (source.size() != target.size()) {
return false;
}
for(int i = 0; i < source.size(); i++) {
if (!source.get(i).equals(target.get(i))) {
return false;
}
}
return true;
}
}