mdrillCommit.java

package com.alimama.mdrillImport;

import java.sql.Date;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.Map.Entry;

import org.apache.log4j.Logger;
import org.apache.solr.common.SolrInputDocument;

import com.alimama.mdrill.adhoc.TimeCacheMap;
import com.alimama.mdrill.ui.service.MdrillService;

public class mdrillCommit implements TimeCacheMap.ExpiredCallback<BoltStatKey,BoltStatVal>{
	private static Logger LOG = Logger.getLogger(mdrillCommit.class);
	private static Timer TIMER_LIST=null;  
	private static Object TIMER_LOCK=new Object();  
    private TimeCacheMap<BoltStatKey, BoltStatVal> group=null;
	private DataParser parse;
	private TimeCacheMap.Update<BoltStatKey, BoltStatVal> update=null;
	private ArrayList<SolrInputDocument> doclist=null;
	private Object doclistLock=new Object();

    private int commitbatch=5000;
    private int buffersize=50000;
	private TimeOutCheck timeoutCheckcommit=null;
    private String confPrefix;

	private int timeout;
	private volatile long lastts[]={0,0};
	private volatile long committs=System.currentTimeMillis();

	public mdrillCommit(DataParser parse,Map conf,String confPrefix)
	{
		synchronized (TIMER_LOCK) {
			if(TIMER_LIST==null)
			{
				TIMER_LIST=new Timer(); 
			}
		}
		
    	this.confPrefix=confPrefix;

		this.parse=parse;
		this.commitbatch=Integer.parseInt(String.valueOf(conf.get(confPrefix+"-commitbatch")));
		this.buffersize=Integer.parseInt(String.valueOf(conf.get(confPrefix+"-commitbuffer")));
		this.timeout=Integer.parseInt(String.valueOf(conf.get(confPrefix+"-timeoutCommit")));
		timeoutCheckcommit=new TimeOutCheck(10*1000l);
		this.group=new TimeCacheMap<BoltStatKey, BoltStatVal>(TIMER_LIST,Math.max(this.timeout, 20), this);
    	this.doclist=new ArrayList<SolrInputDocument>(300);
		this.update=new TimeCacheMap.Update<BoltStatKey, BoltStatVal>() {
    		@Override
    		public synchronized BoltStatVal update(BoltStatKey key, BoltStatVal old,
    				BoltStatVal newval) {
    			if(old==null)
    			{
    				return newval;
    			}
    			BoltStatVal rtn=old;
    			rtn.merger(newval);
    			return rtn;
    		}
    	};
	}
		

	public synchronized void updateAll(HashMap<BoltStatKey, BoltStatVal> buffer)
	{
		this.lastts=this.getMinTs(buffer);
		this.group.updateAll(buffer, this.update);

		group.maybeClean();
		int ramsize=group.size();
		if(ramsize>buffersize)
		{
			LOG.info(this.confPrefix+" fourceTimeout:"+this.toDebugString());
			group.fourceClean();
		}
	}
	
	private long[] getMinTs(HashMap<BoltStatKey, BoltStatVal> buffer)
	{
		long logTs=System.currentTimeMillis();
		long logTsmax=System.currentTimeMillis();

		for(Entry<BoltStatKey, BoltStatVal> e:buffer.entrySet())
		{
			BoltStatKey key=e.getKey();
			BoltStatVal bv=e.getValue();
			
			long ts=bv.getGroupts();
			logTs=Math.min(logTs, ts);
			logTsmax=Math.max(logTsmax, ts);

		}
		return new long[]{logTs,logTsmax};
	}
	


    private static SimpleDateFormat formatHour = new SimpleDateFormat("HH:mm:ss");
    
	@Override
	public synchronized void expire(BoltStatKey key, BoltStatVal val) {
		try{
			this.committs=val.getGroupts();
	 	SolrInputDocument doc=new SolrInputDocument();

		String[] groupnames=parse.getGroupName();
		for(int i=0;i<groupnames.length&&i<key.list.length;i++)
		{
		    doc.addField(groupnames[i], key.list[i]);
		}
		
		String[] statNames=parse.getSumName();

		for(int i=0;i<statNames.length&&i<val.list.length;i++)
		{
		    doc.addField(statNames[i], val.list[i]);
		}
	
	    boolean needCommit=false;

    	synchronized (doclistLock) {
   		 	doclist.add(doc);
   		 	needCommit=doclist.size()>this.commitbatch;
		}
    	
    	if(needCommit||timeoutCheckcommit.istimeout())
    	{
    		timeoutCheckcommit.reset();
		 	this.commit();
    	}
		}catch(Throwable e)
		{
			LOG.info("expire "+this.toDebugString(),e);

		}
	}
	
	long lastPrintTime=System.currentTimeMillis();

	@Override
	public synchronized void commit() {
		try{
		ArrayList<SolrInputDocument> buffer=null;
		synchronized (doclistLock) {
	 		buffer=doclist;
		 	doclist=new ArrayList<SolrInputDocument>(300);
		}

    	if(buffer!=null&&buffer.size()>0)
    	{
	    	for(int i=0;i<100;i++)
	    	{
	    		try {
					LOG.info(this.confPrefix+"@"+this.parse.getTableName()+" mdrill request:"+i+"@"+buffer.size()+","+this.toDebugString());
					long now=System.currentTimeMillis();
					if(now-lastPrintTime>1000l*30)
					{
						lastPrintTime=now;
						LOG.info(this.confPrefix+"@"+this.parse.getTableName()+",doc:"+String.valueOf(buffer.get(0)));

					}
					
					MdrillService.insertLocal(this.parse.getTableName(), buffer,null);
					break ;
				} catch (Throwable e) {
					LOG.error(this.confPrefix+" insert", e);
				}
				try {
					Thread.sleep(1000);
				} catch (InterruptedException e) {
				}
	    	}
    	}
		}catch(Throwable e)
		{
			LOG.info("commit "+this.toDebugString(),e);
		}
	}


	public String toDebugString() {
		try {
		return "access "+formatHour.format(new Date(lastts[0]))+"@"+formatHour.format(new Date(lastts[1]))+"@"+formatHour.format(new Date(this.committs))+",doclist="+doclist.size()+",group="+group.size();
		} catch (Throwable e) {
		}
		return "";
	}
}