ImportBolt.java

package com.alimama.mdrillImport;

import java.util.HashMap;
import java.util.Map;

import org.apache.log4j.Logger;


import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Tuple;

public class ImportBolt implements IRichBolt {
	private static Logger LOG = Logger.getLogger(ImportBolt.class);
    private static final long serialVersionUID = 1L;
    private String confPrefix;

    public ImportBolt(String confPrefix)
    {
    	this.confPrefix=confPrefix;

    }

	private OutputCollector collector=null;
	private HashMap<BoltStatKey, BoltStatVal> nolockbuffer=null;

	private mdrillCommit commit=null;//new LastTimeBolt();
	
	private TimeOutCheck timeoutCheck=null;
	
    private DataParser parse=null;
	private BoltStatus status=null;
    int buffersize=5000;
    int intMapsize=500;
    int checkIntervel=1000;


    @Override
    public void prepare(Map conf, TopologyContext context,
            OutputCollector collector) {
    	try {
			parse=(DataParser) Class.forName(String.valueOf(conf.get(this.confPrefix+"-parse"))).newInstance();
			parse.init(true, conf, context);
		} catch (Throwable e1) {
			LOG.error(this.confPrefix+" DataParser",e1);
		}
    	
		int timeout=Integer.parseInt(String.valueOf(conf.get(confPrefix+"-timeoutBolt")));
		this.buffersize=Integer.parseInt(String.valueOf(conf.get(confPrefix+"-boltbuffer")));
		Object chkIntervel=conf.get(confPrefix+"-boltIntervel");
		if(chkIntervel!=null)
		{
			this.checkIntervel=Integer.parseInt(String.valueOf(chkIntervel));
		}
		
    	this.intMapsize=Math.min(this.buffersize, 1024);

    	this.status=new BoltStatus();

    	this.timeoutCheck=new TimeOutCheck(timeout*1000l);

    	this.commit=new mdrillCommit(parse,conf,this.confPrefix);
    	
    	this.collector=collector;
    	this.nolockbuffer=new HashMap<BoltStatKey, BoltStatVal>(this.intMapsize);
    }
    
	
    @Override
    public synchronized void execute(Tuple input) {
    	this.status.InputCount++;
    	BoltStatKey key=(BoltStatKey) input.getValue(0);
    	BoltStatVal bv=(BoltStatVal)input.getValue(1);
    	
    	

    	BoltStatVal statval=nolockbuffer.get(key);
    	if(statval==null)
    	{
    		this.status.groupCreate++;
    		nolockbuffer.put(key, bv);
    	}else{
    		statval.merger(bv);
    	}
    	
    	if((this.status.InputCount%this.checkIntervel!=0))
    	{
        	this.collector.ack(input);

    		return;
    	}
		boolean isNotOvertime = !this.timeoutCheck.istimeout();
		if (isNotOvertime && nolockbuffer.size() < buffersize) {
	    	this.collector.ack(input);

			return;
		}
    	
	    	HashMap<BoltStatKey, BoltStatVal> buffer=nolockbuffer;
			nolockbuffer=new HashMap<BoltStatKey, BoltStatVal>(this.intMapsize);
			this.commit.updateAll(buffer);
		
		LOG.info(this.confPrefix+" bolt flush:groupsize="+buffer.size()+","+this.status.toString()+","+this.commit.toDebugString());
		this.timeoutCheck.reset();
    	this.collector.ack(input);

    }
    

    @Override
    public void cleanup() {
    	
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
    	
    }

    @Override
    public Map<String, Object> getComponentConfiguration() {
        return new HashMap<String, Object>();
    }

}