package com.leansoft.luxun.perf; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.leansoft.luxun.common.exception.ConsumerTimeoutException; import com.leansoft.luxun.consumer.StreamFactory; import com.leansoft.luxun.consumer.ConsumerConfig; import com.leansoft.luxun.consumer.IStreamFactory; import com.leansoft.luxun.consumer.MessageStream; import com.leansoft.luxun.message.Message; import com.leansoft.luxun.utils.ImmutableMap; import joptsimple.ArgumentAcceptingOptionSpec; import joptsimple.OptionSet; public class ConsumerPerformance { private static Logger logger = LoggerFactory.getLogger(ConsumerPerformance.class); public static void main(String[] args) { try { ConsumerPerfConfig config = ConsumerPerfConfig.fromArgs(args); logger.info("Starting consumer ..."); AtomicLong totalMessagesRead = new AtomicLong(0); AtomicLong totalBytesRead = new AtomicLong(0); if(!config.hideHeader) { if(!config.showDetailedStats) System.out.println("start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec"); else System.out.println("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec"); } IStreamFactory streamFactory = new StreamFactory(config.consumerConfig); Map<String, List<MessageStream<Message>>> topicMessageStreams = streamFactory.createMessageStreams(ImmutableMap.of(config.topic, config.numThreads)); List<ConsumerPerfThread> threadList = new ArrayList<ConsumerPerfThread>(); for(String topic: topicMessageStreams.keySet()) { List<MessageStream<Message>> streamList = topicMessageStreams.get(topic); int i = 0; for (MessageStream<Message> stream : streamList) { threadList.add(new ConsumerPerfThread(i, "luxun-consumer-" + topic, stream, config, totalMessagesRead, totalBytesRead)); i++; } } logger.info("Sleeping for 1000 seconds."); Thread.sleep(1000); logger.info("starting threads"); long startMs = System.currentTimeMillis(); for (ConsumerPerfThread thread : threadList) thread.start(); for (ConsumerPerfThread thread : threadList) thread.shutdown(); streamFactory.close(); long endMs = System.currentTimeMillis(); double elapsedSecs = (endMs - startMs - config.consumerConfig.getConsumerTimeoutMs()) / 1000.0; if (!config.showDetailedStats) { double totalMBRead = (totalBytesRead.get() * 1.0) / (1024 * 1024); System.out.println(String.format("%s, %s, %d, %.4f, %.4f, %d, %.4f", config.dateFormat.format(startMs), config.dateFormat.format(endMs), config.consumerConfig.getFetchSize(), totalMBRead, totalMBRead / elapsedSecs, totalMessagesRead.get(), totalMessagesRead.get() / elapsedSecs)); } System.exit(0); } catch (Exception e) { e.printStackTrace(); System.exit(-1); } } static class ConsumerPerfThread extends Thread { private final int threadId; private final MessageStream<Message> stream; private final ConsumerPerfConfig config; private final AtomicLong totalMessagesRead; private final AtomicLong totalBytesRead; private CountDownLatch shutdownLatch = new CountDownLatch(1); public void shutdown() { try { this.shutdownLatch.await(); } catch (InterruptedException e) { // ignore } } public ConsumerPerfThread( int threadId, String name, MessageStream<Message> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead ) { super(name); this.threadId = threadId; this.stream = stream; this.config = config; this.totalMessagesRead = totalMessagesRead; this.totalBytesRead = totalBytesRead; } @Override public void run() { long bytesRead = 0L; long messagesRead = 0L; long startMs = System.currentTimeMillis(); long lastReportTime = startMs; long lastBytesRead = 0L; long lastMessagesRead = 0L; try { for(Message message : stream) { messagesRead += 1; bytesRead += message.length(); if (messagesRead % config.reportingInterval == 0) { if (config.showDetailedStats) { this.printMessage(threadId, bytesRead, lastBytesRead, messagesRead, lastMessagesRead, lastReportTime, System.currentTimeMillis()); } lastReportTime = System.currentTimeMillis(); lastMessagesRead = messagesRead; lastBytesRead = bytesRead; } } } catch (ConsumerTimeoutException te) { logger.info("No more to consume, consumer thread begin to exit ..."); } totalMessagesRead.addAndGet(messagesRead); totalBytesRead.addAndGet(bytesRead); if (config.showDetailedStats) { this.printMessage(threadId, bytesRead, lastBytesRead, messagesRead, lastMessagesRead, startMs, System.currentTimeMillis()); } this.shutdownLatch.countDown(); } private void printMessage(int id, long bytesRead, long lastBytesRead, long messagesRead, long lastMessagesRead, long startMs, long endMs) { long elapsedMs = endMs - startMs; double totalMBRead = (bytesRead * 1.0) / (1024 * 1024); double mbRead = ((bytesRead - lastBytesRead) * 1.0) / (1024 * 1024); System.out.println(String.format("%s, %d, %d, %.4f, %.4f, %d, %.4f", config.dateFormat.format(endMs), id, config.consumerConfig.getFetchSize(), totalMBRead, 1000.0 * (mbRead/elapsedMs), messagesRead, ((messagesRead - lastMessagesRead) / elapsedMs) * 1000.0)); } } static class ConsumerPerfConfig extends PerfConfig { protected static ArgumentAcceptingOptionSpec<String> brokerInfoOpt = parser.accepts("brokerinfo", "REQUIRED: broker info list to consume from." + "Multiple brokers can be given to allow concurrent concuming") .withRequiredArg(). describedAs("brokerid1:hostname1:port1,brokerid2:hostname2:port2") .ofType(String.class); protected static ArgumentAcceptingOptionSpec<String> groupIdOpt = parser.accepts("group", "The group to consume on.") .withRequiredArg() .describedAs("gid") .defaultsTo("perf-consumer-" + new Random().nextInt(100000)) .ofType(String.class); protected static ArgumentAcceptingOptionSpec<Integer> fetchSizeOpt = parser.accepts("fetch-size", "The amount of data to fetch in a single request.") .withRequiredArg(). describedAs("size") .ofType(Integer.class) .defaultsTo(1024 * 1024); protected static ArgumentAcceptingOptionSpec<Integer> numThreadsOpt = parser.accepts("threads", "Number of processing threads.") .withRequiredArg(). describedAs("count"). ofType(Integer.class). defaultsTo(10); ConsumerConfig consumerConfig; int numThreads; public static ConsumerPerfConfig fromArgs(String[] args) throws Exception { OptionSet options = parser.parse(args); checkRequiredArgs(options, topicOpt, brokerInfoOpt); ConsumerPerfConfig config = new ConsumerPerfConfig(); fillCommonConfig(options, config); Properties props = new Properties(); props.put("groupid", options.valueOf(groupIdOpt)); props.put("fetch.size", String.valueOf(options.valueOf(fetchSizeOpt))); props.put("broker.list", options.valueOf(brokerInfoOpt)); props.put("consumer.timeout.ms", "5000"); config.consumerConfig = new ConsumerConfig(props); config.numThreads = options.valueOf(numThreadsOpt).intValue(); return config; } } }