SimpleConsumerPerformance.java

package com.leansoft.luxun.perf;

import java.net.URI;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.leansoft.luxun.api.generated.apiConstants;
import com.leansoft.luxun.consumer.SimpleConsumer;
import com.leansoft.luxun.message.Message;
import com.leansoft.luxun.message.MessageList;

import joptsimple.ArgumentAcceptingOptionSpec;
import joptsimple.OptionSet;
import joptsimple.OptionSpecBuilder;

public class SimpleConsumerPerformance {
	
	private static final Logger logger = LoggerFactory.getLogger(SimpleConsumerPerformance.class);
	
    public static void main(String[] args) {
    	try {
    		
    		ConsumerPerfConfig config = ConsumerPerfConfig.fromArgs(args);
    		
    		if (!config.hideHeader) {
    			if (!config.showDetailedStats) {
    				System.out.println("start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
    			} else {
    				System.out.println("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
    			}
    		}
    		
    		SimpleConsumer consumer = new SimpleConsumer(config.url.getHost(), config.url.getPort(), 30 * 1000);
    		long timeStamp = config.fromLatest ? apiConstants.LATEST_TIME : apiConstants.EARLIEST_TIME;
    		long index = consumer.findClosestIndexByTime(config.topic, timeStamp);
    		logger.info("Consumer got " + (config.fromLatest ? apiConstants.LATEST_INDEX_STRING : apiConstants.EARLIEST_INDEX_STRING) + " index : " + index);
    		
    		long startMs = System.currentTimeMillis();
    		boolean done = false;
    		long totalBytesRead = 0L;
    		long totalMessagesRead = 0L;
    		int consumedInterval = 0;
    		long lastReportTime = startMs;
    		long lastBytesRead = 0L;
    		long lastMessagesRead = 0L;
    		
    		while(!done) {
    			List<MessageList> listOfMessageList = consumer.consume(config.topic, index, config.fetchSize);
    			
    			int messagesRead = 0;
    			int bytesRead = 0;
    			
    			for(MessageList messageList : listOfMessageList) {
    				
    				for(Message message : messageList) {
    					bytesRead += message.length();
    				}
    				
    				messagesRead += messageList.size();
    			}
    			
    			if (messagesRead == 0 || totalMessagesRead > config.numMessages) {
    				done = true;
    			} else {
    				index += listOfMessageList.size();
    			}
    			
    			totalBytesRead += bytesRead;
    			totalMessagesRead += messagesRead;
    			consumedInterval += messagesRead;
    			
    			if (consumedInterval > config.reportingInterval) {
    				if (config.showDetailedStats) {
    					long reportTime = System.currentTimeMillis();
    					double elapsed = (reportTime - lastReportTime) / 1000.0;
    					double totalMBRead = ((totalBytesRead - lastBytesRead) * 1.0) / (1024 * 1024);
    					System.out.println(String.format("%s, %d, %.4f, %.4f, %d, %.4f", config.dateFormat.format(reportTime), config.fetchSize,
    							(totalBytesRead * 1.0) / (1024 * 1024), totalMBRead / elapsed,
    							totalMessagesRead, (totalMessagesRead - lastMessagesRead) / elapsed
    							));
    				}
    				lastReportTime = System.currentTimeMillis();
    				lastBytesRead = totalBytesRead;
    				lastMessagesRead = totalMessagesRead;
    				consumedInterval = 0;
    			}
    		}
    		
    		long reportTime = System.currentTimeMillis();
    		double elapsed = (reportTime - startMs) / 1000.0;
    		
    		if (!config.showDetailedStats) {
    			double totalMBRead = (totalBytesRead * 1.0) / (1024 * 1024);
    			System.out.println(String.format("%s, %s, %d, %.4f, %.4f, %d, %.4f", config.dateFormat.format(startMs),
    					config.dateFormat.format(reportTime), config.fetchSize, totalMBRead, totalMBRead/elapsed,
    					totalMessagesRead, totalMessagesRead / elapsed
    					));
    		}
    		
    		consumer.close();
    		
    		System.exit(0);
    		
		} catch (Exception e) {
			e.printStackTrace();
			System.exit(-1);
		}	
    }	
	
	static class ConsumerPerfConfig extends PerfConfig {
		
		protected static ArgumentAcceptingOptionSpec<String> urlOpt = parser.accepts("server", "REQUIRED: The hostname of the server to connect to.")
		        .withRequiredArg().
		        describedAs("luxun://hostname:port")
		        .ofType(String.class);
		
		protected static OptionSpecBuilder resetBeginningIndexOpt = parser.accepts("from-latest", "If the consumer does not already have an established " +
		   "index to consume from, start with the latest message present in the log rather than the earliest message.");
		
		
		protected static ArgumentAcceptingOptionSpec<Integer> fetchSizeOpt = parser.accepts("fetch-size", "The fetch size to use for consumption.")
		        .withRequiredArg().
		        describedAs("bytes")
		        .ofType(Integer.class)
		        .defaultsTo(1024 * 1024);
		
		URI url;
		int fetchSize;
		boolean fromLatest;
		int partition;
		
		
		public static ConsumerPerfConfig fromArgs(String[] args) throws Exception {
	        OptionSet options = parser.parse(args);
	        checkRequiredArgs(options, topicOpt, urlOpt);
	        
	        ConsumerPerfConfig config = new ConsumerPerfConfig();
	        
	        fillCommonConfig(options, config);
	        
	        config.url = new URI(options.valueOf(urlOpt));
	        config.fetchSize = options.valueOf(fetchSizeOpt).intValue();
	        config.fromLatest = options.has(resetBeginningIndexOpt);
	        
	        return config;
	       
		}
		
	}

}