package com.senseidb.gateway.perf;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import kafka.message.Message;

import com.senseidb.gateway.kafka.DefaultJsonDataSourceFilter;

public class PrimeKafkaUtil {

  static Charset UTF8 = Charset.forName("UTF-8");
  /**
   * @param args
   */
  public static void main(String[] args) throws Exception{
    File f = new File("/home/jwang/github/search-perf/data/cars1m.json");
    
    Properties props = new Properties();
    props.put("zookeeper.connect", "localhost:2181");
    props.put("serializer.class", "kafka.serializer.DefaultEncoder");

    ProducerConfig producerConfig = new ProducerConfig(props);
    Producer<String,Message> kafkaProducer = new Producer<String,Message>(producerConfig);
    String topic = "perfTopic";
    List<KeyedMessage<String, Message>> msgList = new ArrayList<KeyedMessage<String, Message>>();
   
    
    BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(f),UTF8));
    
    int batchSize = 10000;
    int count = 0;
    while(true){
      String line = reader.readLine();
      if (line==null) break;
      count++;
      System.out.println(count+" msgs pushed.");
      Message m = new Message(line.getBytes(DefaultJsonDataSourceFilter.UTF8));
        KeyedMessage<String,Message> msg = new KeyedMessage<String,Message>(topic,m);
      msgList.add(msg);

      if (msgList.size()>batchSize){
        kafkaProducer.send(msgList);
        msgList.clear();
      }
    }

    if (msgList.size()>0){
      kafkaProducer.send(msgList);
    }
  }

}