MdrillDetail.java

package org.apache.solr.request.mdrill;

import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.zip.CRC32;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.MapFieldSelector;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.queryParser.ParseException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.DocSetCollector;
import org.apache.solr.search.SolrIndexSearcher;

import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.compare.ColumnKey;
import org.apache.solr.request.compare.RecordCountDetail;
import org.apache.solr.request.compare.SelectDetailRow;
import org.apache.solr.request.join.HigoJoinInvert;
import org.apache.solr.request.mdrill.MdrillUtils.*;

import com.alimama.mdrill.utils.EncodeUtils;
import com.alimama.mdrill.utils.UniqConfig;

/**
	查询明细的实现
 * @author yannian.mu
 */
public class MdrillDetail {
//	public static Integer MAX_CROSS_ROWS=UniqConfig.defaultCrossMaxLimit();
	private SolrIndexSearcher searcher;
	private RecordCountDetail recordCount ;
	private SolrQueryRequest req;
	private MdrillParseDetail parse;
	private SegmentReader reader;
	private MdrillParseDetail.fetchContaioner container=null;
	
	String crcget=null;
	String segKey=null;
	private String hostId=java.util.UUID.randomUUID().toString();
	SolrParams params;

	public MdrillDetail(SolrIndexSearcher _searcher,SegmentReader reader,SolrParams _params,SolrQueryRequest req)
	{
		try {
			this.hostId=java.net.InetAddress.getLocalHost().getHostAddress()+","+String.valueOf(java.net.InetAddress.getLocalHost().getHostName());
		} catch (UnknownHostException e) {
			hostId=java.util.UUID.randomUUID().toString();
		}
		
		this.segKey=reader.getSigmentUniqKey();
		this.crcget=_params.get("mdrill.crc.key.get",null);
		
		this.reader=reader;
		this.req=req;
		this.searcher=_searcher;
		this.parse=new MdrillParseDetail(_params);
		this.recordCount = new RecordCountDetail();
		this.params=_params;

	}
	
	public static class FieldValueGet{
		String[] fields;
		SegmentReader reader;
		SolrIndexSearcher searcher;
		boolean[] isStore;
		FieldType[] ftlist;
		MapFieldSelector selector;
		String[] fieldsnostore;
		UnvertFields ufs;
		public FieldValueGet(DocSet baseAdvanceDocs,String[] fields,SegmentReader reader,SolrIndexSearcher searcher) throws IOException
		{
			this.fields=fields;
			this.reader=reader;
			this.searcher=searcher;
			this.isStore=new boolean[this.fields.length];
			this.fieldsnostore=new String[this.fields.length];
			
			this.ftlist=new FieldType[fields.length];
			IndexSchema schema=this.searcher.getSchema();

			ArrayList<String> storedField=new ArrayList<String>();
			
			for (int j = 0; j < fields.length; j++) {
				ftlist[j]=schema.getFieldType(fields[j]);
				this.isStore[j]=schema.getField(fields[j]).stored();
				this.fieldsnostore[j]=fields[j];
				if(this.isStore[j])
				{
					storedField.add(fields[j]);
					this.fieldsnostore[j]="higoempty_"+j+"_s";
				}
				
			}
			this.selector=new MapFieldSelector(storedField); 
			this.ufs=new UnvertFields(baseAdvanceDocs,this.fieldsnostore, reader,searcher.getPartionKey(),searcher.getSchema(),false);;
			
		}
		
		public void free()
		{
			this.ufs.free();
		}
		public String[] doc(int docid) throws CorruptIndexException, IOException
		{
			Document docfields=this.reader.document(docid,selector);

			String[] rtn=new String[this.fields.length];
			for (int j = 0; j < fields.length; j++) {
				rtn[j]="-";
				String field=fields[j];
				if(this.isStore[j])
				{
					Fieldable fv=docfields.getFieldable(field);
					if(fv!=null)
					{
						rtn[j]=ftlist[j].toExternal(fv);
					}
				}else{
					UnvertFile uf=ufs.cols[j];
					if(uf!=null)
					{	
						Integer termNum = uf.uif.termNum(docid);
						 String fieldvalue=uf.uif.tNumToString(termNum, uf.filetype, uf.ti,"null");
						 if(fieldvalue!=null)
						 {
							 rtn[j]=fieldvalue;
						 }
					}
				}
			}
			
			return rtn;
		}
	
	}
	

	public NamedList get(String[] fields, DocSet baseDocs) throws IOException,
			ParseException {
		if(this.crcget==null)
		{
		
			this.container=this.parse.createContainer(fields, baseDocs, this.reader, this.searcher, this.req);
			DocIterator iter = baseDocs.iterator();
			int doc=-1;
			if(this.container.isUseJoinSort())
			{
				int cmpValue=0;
				while (iter.hasNext()) {
					doc = iter.nextDoc();
					if(!this.container.containsInJoins(doc))
					{
						continue;
					}
					cmpValue=this.container.getJoinCompareValue(doc);
					SelectDetailRow row = SelectDetailRow.INSTANCE(doc,cmpValue);
					QueuePutUtils.put2QueueDetail(row, this.container.res, this.parse.limit_offset, this.container.cmpTermNum);
					this.recordCount.inc(1);
				}
			}else if(this.container.isColumnSort())
			{
				int cmpValue=0;
				
					while (iter.hasNext()) {
						doc = iter.nextDoc();
						if(!this.container.containsInJoins(doc))
						{
							continue;
						}
						cmpValue =this.container.getColumnCompareValue(doc);
						SelectDetailRow row = SelectDetailRow.INSTANCE(doc,cmpValue);
						QueuePutUtils.put2QueueDetail(row, this.container.res, this.parse.limit_offset, this.container.cmpTermNum);
						this.recordCount.inc(1);
					}
			}else{
				while (iter.hasNext()) {
					doc = iter.nextDoc();
					if(!this.container.containsInJoins(doc))
					{
						continue;
					}
					SelectDetailRow row = SelectDetailRow.INSTANCE(doc,doc);
					QueuePutUtils.put2QueueDetail(row, this.container.res, this.parse.limit_offset, this.container.cmpTermNum);
					this.recordCount.inc(1);
					if(this.container.res.size()>=this.parse.limit_offset)
					{
						break;
					}
				}
				while (iter.hasNext()) {
					doc = iter.nextDoc();
					if(!this.container.containsInJoins(doc))
					{
						continue;
					}
					this.recordCount.inc(1);
				}
			}
			
			PriorityQueue<SelectDetailRow> topItems=this.transGroupValue(this.container.res,this.container.sortufs);
					
			 this.container.free();
			
			return this.toNameList(topItems);	
		
		}
		
		
		String hostkey=String.valueOf(this.getkeyCrc());

		ConcurrentHashMap<Long,String> cache=MdrillUtils.CRC_CACHE_SIZE.remove(crcget+"@"+hostkey);
		NamedList rtn=new NamedList();
		Map<Long,String> crcvalue=new HashMap<Long,String>();
		rtn.add("fdtcre", crcvalue);
		if(cache!=null)
		{

			String crcliststr=params.get("mdrill.crc.key.get.crclist");
			if(crcliststr!=null)
			{
				

				DocSetCollector collect=new DocSetCollector(10240, reader.maxDoc());

				String[] crclist=crcliststr.split(",");
				for(String s:crclist)
				{
					Long crc=Long.parseLong(s);
					String v=cache.get(crc);
					if(v!=null)
					{
						String cols[]=v.split(UniqConfig.GroupJoinString(),-1);
						if(cols.length>=2)
						{
							int doc=Integer.parseInt(cols[0]);
							collect.collect(doc);
						}
					}
				}
				
				
				FieldValueGet fvget=new FieldValueGet(collect.getDocSet(),fields, reader, searcher);

				for(String s:crclist)
				{
					Long crc=Long.parseLong(s);
					String v=cache.get(crc);
					if(v!=null)
					{
						String cols[]=v.split(UniqConfig.GroupJoinString(),-1);
						if(cols.length>=2)
						{
							int doc=Integer.parseInt(cols[0]);
							
							String[] fieldValue=fvget.doc(doc);
							SortGroupVal buff = new SortGroupVal();
							buff.groupbuff.append("-");
							buff.groupbuff.append(UniqConfig.GroupJoinString());
							buff.groupbuff.append("-");
							for (int j = 0; j < fields.length; j++) {
								buff.groupbuff.append(UniqConfig.GroupJoinString());
								buff.groupbuff.append(EncodeUtils.encode(fieldValue[j]));
							}
							
							for(int j=(fields.length+2);j<cols.length;j++)
							{
								buff.groupbuff.append(UniqConfig.GroupJoinString());
								buff.groupbuff.append(cols[j]);
							}
							
							crcvalue.put(crc, buff.groupbuff.toString());

						}
					}
				}
				fvget.free();

			}
			
		}
		return rtn;
	}

	
	
	private TermNumToString prefetchValues(PriorityQueue<SelectDetailRow> res,UnvertFields sortufs) throws IOException
	{
		TermNumToString forsort=new TermNumToString(sortufs,0);
		boolean usedsort=this.container.isOnlyColumnSort();

		for (SelectDetailRow row : res) {
			int doc = row.docid;
				
			if(usedsort)
			{
				int ternnum=(int)row.value;
				forsort.addTermNum(ternnum);
			}
			
			
			for(HigoJoinInvert inv:this.container.joinInvert)
			{
				int fc=inv.fieldCount();
				LinkedBlockingQueue<GroupListCache.GroupList> groupListCache=inv.getGroupListCache();
				GroupListCache.GroupList base=GroupListCache.GroupList.INSTANCE(groupListCache, fc);
				base.reset();
				if(inv.fieldNum(doc, base))
				{
					for (int i = 0; i < fc; i++) {
						inv.addTermNum(base.list[i], i);
					}
				}

				groupListCache.add(base);
			}
		}
		
		
		forsort.fetchValues();
		 
		 for(HigoJoinInvert inv:this.container.joinInvert)
		 {
			 inv.fetchValues();
		 }
		 
		 return forsort;
	}

	public PriorityQueue<SelectDetailRow> transGroupValue(PriorityQueue<SelectDetailRow> res,UnvertFields sortufs)throws ParseException, IOException {
		TermNumToString tm=this.prefetchValues( res, sortufs);
		
		PriorityQueue<SelectDetailRow> topItems = new PriorityQueue<SelectDetailRow>(this.parse.limit_offset, Collections.reverseOrder(this.container.cmpresult));

		boolean usedsort=this.container.isOnlyColumnSort();

		long key=getkeyCrc();

		for (SelectDetailRow row : res) {
			int doc = row.docid;
			String sortString=null;
			if(usedsort)
			{
				sortString=tm.getTermValueWithNull((int)row.value);
			}
			SortGroupVal buff = new SortGroupVal();
			buff.groupbuff.append(String.valueOf(doc));
			buff.groupbuff.append(UniqConfig.GroupJoinString());
			buff.groupbuff.append(String.valueOf(key));
			for (int i = 0; i < this.container.fields.length; i++) {
				buff.groupbuff.append(UniqConfig.GroupJoinString());
				buff.groupbuff.append(EncodeUtils.encode("-"));
			}
			buff.sortString=sortString;
			this.setGroupJoin(buff,this.container.fields.length,doc);
			SelectDetailRow newrow = SelectDetailRow.INSTANCE(doc, row.getCompareValue());
			newrow.setKey(new ColumnKey(buff.groupbuff.toString()));
			newrow.colVal=buff.sortString;
			QueuePutUtils.put2QueueDetail(newrow, topItems, this.parse.limit_offset, this.container.cmpresult);
		}

		return topItems;
	}
	
	
	public void setGroupJoin(SortGroupVal buff,int offset,int doc) throws IOException
	{
		for(HigoJoinInvert inv:this.container.joinInvert)
		{
			int fc=inv.fieldCount();
			LinkedBlockingQueue<GroupListCache.GroupList> groupListCache=inv.getGroupListCache();
			GroupListCache.GroupList base=GroupListCache.GroupList.INSTANCE(groupListCache, fc);
			base.reset();
			if(inv.fieldNum(doc, base))
			{

				for (int i = 0; i < fc; i++) {
					buff.groupbuff.append(UniqConfig.GroupJoinString());
					buff.groupbuff.append(EncodeUtils.encode(inv.getTermNumValue(base.list[i], i)));
					if(this.container.SelectDetailSort!=null&&(offset+i)==this.container.SelectDetailSort.offset)
					{
						buff.sortString=inv.getTermNumValue(base.list[this.container.SelectDetailSort.selfOffset], i);
					}
				}
			}
			groupListCache.add(base);
			offset+=fc;
		}
	}
	
	

	private NamedList toNameList(PriorityQueue<SelectDetailRow> topItems) {
		java.util.ArrayList<SelectDetailRow> recommendations = new ArrayList<SelectDetailRow>(topItems.size());
		recommendations.addAll(topItems);
		Collections.sort(recommendations, this.container.cmpresult);
		Integer index = 0;
		NamedList res = new NamedList();
		res.add("count", recordCount.toNamedList());
		
		ConcurrentHashMap<Long,String> cache=null;
		boolean issetCrc=this.parse.crcOutputSet!=null;
		long crckey=this.getkeyCrc();
		

		if(issetCrc)
		{	
			synchronized (MdrillUtils.CRC_CACHE_SIZE) {
				cache=MdrillUtils.CRC_CACHE_SIZE.get(this.parse.crcOutputSet+"@"+crckey);
				if(cache==null)
				{
					cache=new ConcurrentHashMap<Long,String>();
					MdrillUtils.CRC_CACHE_SIZE.put(this.parse.crcOutputSet+"@"+crckey, cache);
				}
			}
			
			
		}
		
		ArrayList<Object> list=new ArrayList<Object>();
		for (SelectDetailRow kv : recommendations) {
			if (index >= this.parse.offset) {
				if(issetCrc)
				{
					kv.ToCrcSet(cache);
				}
				list.add(kv.toNamedList());
			}
			index++;
		}
		res.add("list", list);
		return res;
	}
	
	
	
	public static class SortGroupVal{
		public StringBuffer groupbuff=new StringBuffer();
		public String sortString;
	}
	
	private long getkeyCrc()
	{
		CRC32 crc32 = new CRC32();
		crc32.update(String.valueOf(hostId+"@"+this.segKey).getBytes());
		return crc32.getValue();
	}
	
}