ImportSpoutLocal.java

package com.alimama.mdrillImport;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.apache.solr.common.SolrInputDocument;

import com.alimama.mdrill.ui.service.MdrillService;


import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichSpout;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;

public class ImportSpoutLocal implements IRichSpout{
	private static Logger LOG = Logger.getLogger(ImportSpoutLocal.class);

    private static final long serialVersionUID = 1L;
    private ImportReader reader=null;
    private DataParser parse=null;
    private String confPrefix;

    public ImportSpoutLocal(String confPrefix)
    {
    	this.confPrefix=confPrefix;
    }
	private ArrayList<SolrInputDocument> doclist=null;
	private Object doclistLock=null;


    int buffersize=5000;
    @Override
    public void open(Map conf, TopologyContext context,SpoutOutputCollector collector) {
    	doclistLock=new Object();
    	try {
			parse=(DataParser) Class.forName(String.valueOf(conf.get(this.confPrefix+"-parse"))).newInstance();
			parse.init(true, conf, context);
		} catch (Throwable e1) {
			LOG.error(this.confPrefix+ " DataParser",e1);
		}

		int timeout=Integer.parseInt(String.valueOf(conf.get(confPrefix+"-timeoutSpout")));
		this.buffersize=Integer.parseInt(String.valueOf(conf.get(confPrefix+"-spoutbuffer")));

		Object chkIntervel=conf.get(confPrefix+"-spoutIntervel");
		if(chkIntervel!=null)
		{
			this.checkIntervel=Integer.parseInt(String.valueOf(chkIntervel));
		}
		
    	this.timeoutCheck=new TimeOutCheck(timeout*1000l);
    	this.status=new SpoutStatus();
    	
    	
	 	doclist=new ArrayList<SolrInputDocument>(this.buffersize);

	 	int readerCount=context.getComponentTasks(context.getThisComponentId()).size();
    	int readerIndex=context.getThisTaskIndex();
    	try {
			this.reader=new ImportReader(conf, confPrefix, parse, readerIndex, readerCount);
		} catch (IOException e) {
			LOG.error("TTReader",e);
		}
    }
    

	
	private SpoutStatus status=null;
	private TimeOutCheck timeoutCheck=null;
	

    int checkIntervel=1000;


    
    
	private boolean putdata(DataParser.DataIter log)
	{	
    	long ts=log.getTs();
		BoltStatKey key=new BoltStatKey(log.getGroup());
		BoltStatVal val=new BoltStatVal(log.getSum(),ts);
		
		this.status.ttInput++;
		this.status.groupCreate++;
		
		SolrInputDocument doc=new SolrInputDocument();

		String[] groupnames=parse.getGroupName();
		for(int i=0;i<groupnames.length&&i<key.list.length;i++)
		{
		    doc.addField(groupnames[i], key.list[i]);
		}
		
		String[] statNames=parse.getSumName();

		for(int i=0;i<statNames.length&&i<val.list.length;i++)
		{
		    doc.addField(statNames[i], val.list[i]);
		}
		
		synchronized (doclistLock) {
			doclist.add(doc);
		}

    	if((this.status.ttInput%checkIntervel!=0))
    	{
    		return false;
    	}
    	

    	this.commit();
	    
	    return true;
	}
	
	
	public synchronized void commit() {
		
		synchronized (doclistLock) {
	    	if(!this.timeoutCheck.istimeout()&&doclist.size()<buffersize)
			{
	    		return ;
			}
		}
    	
    	this.timeoutCheck.reset();
		
		try{
		ArrayList<SolrInputDocument> buffer=null;
		synchronized (doclistLock) {
	 		buffer=doclist;
		 	doclist=new ArrayList<SolrInputDocument>(300);
		}

    	if(buffer!=null&&buffer.size()>0)
    	{
	    	for(int i=0;i<100;i++)
	    	{
	    		try {
					LOG.info(this.confPrefix+" mdrill request:"+i+"@"+buffer.size());
					MdrillService.insertLocal(this.parse.getTableName(), buffer,null);
					break ;
				} catch (Throwable e) {
					LOG.error(this.confPrefix+" insert", e);
				}
				try {
					Thread.sleep(1000);
				} catch (InterruptedException e) {
				}
	    	}
    	}
		}catch(Throwable e)
		{
			LOG.info("commit ",e);
		}
	}
	
	
	   @Override
	    public synchronized void nextTuple()  {
			try {
				List ttdata = this.reader.read();
				if(ttdata==null)
				{
					return ;
				}
				for(Object o:ttdata)
				{
					this.status.ttInput++;
					DataParser.DataIter pv=(DataParser.DataIter)o;
					while(true)
					{
						this.status.groupInput++;
						this.putdata(pv);
						if(!pv.next())
						{
							break;
						}
					}
				}
				
			} catch (Throwable e) {
				this.sleep(100);
			}
	    }
	    
	    private void sleep(int i)
	    {
	    	try {
				Thread.sleep(i);
			} catch (InterruptedException e1) {
			}
	    }

	    @Override
	    public void close() {
		
	    }
	        @Override
	    public void ack(Object msgId) {
	        	this.status.ackCnt++;
	    }

	    @Override
	    public void fail(Object msgId) {
	    	this.status.failCnt++;
	    }


    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("key","value"));
    }

    @Override
    public Map<String, Object> getComponentConfiguration() {
        Map<String, Object> rtnmap = new HashMap<String, Object>();
        return rtnmap;
    }

}