/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.request.uninverted;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SegmentReader;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.*;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.cache.Cache;
import org.apache.solr.request.uninverted.GrobalCache.ILruMemSizeCache;
import org.apache.solr.request.uninverted.GrobalCache.ILruMemSizeKey;
import org.apache.solr.request.uninverted.UnInvertedFieldUtils.*;
import com.alimama.mdrill.buffer.LuceneUtils;
import com.alimama.mdrill.utils.UniqConfig;
import java.io.IOException;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class UnInvertedField extends UnInvertedFieldBase {
public static Logger log = LoggerFactory.getLogger(UnInvertedField.class);
public static ThreadPoolExecutor SUBMIT_POOL = new ThreadPoolExecutor(UniqConfig.getUnivertedFieldThreads(),UniqConfig.getUnivertedFieldThreads(), 100L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
public static UnInvertedField getUnInvertedField(DocSet baseAdvanceDocs,String field, SolrIndexSearcher searcher) throws IOException {
return getUnInvertedField(baseAdvanceDocs,field, searcher.getReader());
}
public static BitDocSet ajustBase(int times,BitDocSet baseAdvanceDocs,IndexReader reader)
{
try {
if(baseAdvanceDocs==null)
{
return null;
}
int maxdoc=reader.maxDoc();
int oversize=maxdoc/times;
int size=baseAdvanceDocs.size();
int maxinterval=reader.getMaxInterval();
log.info("ajustBase "+maxinterval+",baseAdvanceDocs="+size+"@"+oversize+"@"+maxdoc+","+reader.getClass().getCanonicalName());
if(size>=oversize||maxinterval>256)
{
return null;
}
} catch (Exception e) {
return null;
}
return baseAdvanceDocs;
}
public static BitDocSet cloneBitset(DocSet baseAdvanceDocs,IndexReader reader)
{
if(baseAdvanceDocs instanceof BitDocSet)
{
BitDocSet rtn= (BitDocSet)baseAdvanceDocs;
OpenBitSet newbits = (OpenBitSet)(rtn.getBits().clone());
return new BitDocSet(newbits,rtn.size());
}
OpenBitSet bs = new OpenBitSet(reader.maxDoc());
DocIterator iter = baseAdvanceDocs.iterator();
int pos=0;
while (iter.hasNext()) {
bs.fastSet(iter.nextDoc());
pos++;
}
return new BitDocSet(bs,pos);
}
public static UnInvertedField getUnInvertedField(final DocSet baseAdvanceDocs,final String field,final SegmentReader reader,String partion,final IndexSchema schema,final boolean isreadDouble) throws IOException
{
final ILruMemSizeKey key = new GrobalCache.StringKey("seg@"+String.valueOf(isreadDouble) + "@" + field + "@"+reader.getStringCacheKey()+"@"+reader.getSegmentName());
ExecutorCompletionService<UnivertPool> submit=new ExecutorCompletionService<UnivertPool>(SUBMIT_POOL);
final long t0=System.currentTimeMillis();
Callable<UnivertPool> task = new Callable<UnivertPool>() {
public UnivertPool call() throws Exception {
UnivertPool rtnuif=new UnivertPool();
try{
long t1=System.currentTimeMillis();
Cache<ILruMemSizeKey, ILruMemSizeCache> cache = GrobalCache.fieldValueCache;
final Object lockthr=UnInvertedFieldUtils.getLock(key);
synchronized (lockthr) {
rtnuif.uni = (UnInvertedField)cache.get(key);
BitDocSet clonebitset=cloneBitset(baseAdvanceDocs,reader);
long t2=System.currentTimeMillis();
if (rtnuif.uni == null||rtnuif.uni.isShutDown()) {
rtnuif.uni = new UnInvertedField();
boolean issucecess=MakeUnivertedFieldBySigment.makeInit(rtnuif.uni,clonebitset,field, reader,schema,isreadDouble);
if(!issucecess)
{
MakeUnivertedFieldByIndex forjoin=new MakeUnivertedFieldByIndex(rtnuif.uni);
forjoin.makeInit(clonebitset, field, schema, reader);
}
cache.put(key, rtnuif.uni);
}else{
boolean issucecess=MakeUnivertedFieldBySigment.addDoclist(rtnuif.uni,clonebitset,field, reader,schema,isreadDouble);
if(!issucecess)
{
MakeUnivertedFieldByIndex forjoin=new MakeUnivertedFieldByIndex(rtnuif.uni);
forjoin.addDoclist(clonebitset, field, reader);
}
}
long t3=System.currentTimeMillis();
log.info("####timetaken####:"+(t3-t2)+"@"+(t2-t1)+"@"+(t1-t0)+","+String.valueOf(rtnuif.uni));
}
}catch(IOException e){
rtnuif.e=e;
}
return rtnuif;
}
};
submit.submit(task);
UnInvertedField uif=UnInvertedFieldUtils.takeUnf(submit);
uif.refCnt.incrementAndGet();
long t4=System.currentTimeMillis();
log.info("####timetaken all####:"+(t4-t0)+","+String.valueOf(uif));
return uif;
}
public static UnInvertedField getUnInvertedField(final DocSet baseAdvanceDocs,final String field,
SolrIndexReader reader) throws IOException {
final SolrIndexSearcher searcher = reader.getSearcher();
final ILruMemSizeKey key = new GrobalCache.StringKey(searcher.getPartionKey() + "@@" + field + "@@" + LuceneUtils.crcKey(reader));
final long t0=System.currentTimeMillis();
ExecutorCompletionService<UnivertPool> submit=new ExecutorCompletionService<UnivertPool>(SUBMIT_POOL);
Callable<UnivertPool> task = new Callable<UnivertPool>() {
public UnivertPool call() throws Exception {
UnivertPool rtnuif=new UnivertPool();
try{
final Cache<ILruMemSizeKey, ILruMemSizeCache> cache = GrobalCache.fieldValueCache;
final Object lockthr=UnInvertedFieldUtils.getLock(key);
synchronized (lockthr) {
long t1=System.currentTimeMillis();
rtnuif.uni = (UnInvertedField)cache.get(key);
SolrIndexReader reader=searcher.getReader();
BitDocSet clonebitset=cloneBitset(baseAdvanceDocs,reader);
if (rtnuif.uni == null||rtnuif.uni.isShutDown()) {
rtnuif.uni = new UnInvertedField();
MakeUnivertedFieldByIndex forjoin=new MakeUnivertedFieldByIndex(rtnuif.uni);
forjoin.makeInit(clonebitset, field, searcher.getSchema(), reader);
cache.put(key, rtnuif.uni);
}else{
MakeUnivertedFieldByIndex forjoin=new MakeUnivertedFieldByIndex(rtnuif.uni);
forjoin.addDoclist(clonebitset, field, reader);
}
long t2=System.currentTimeMillis();
log.info("timetaken by index:"+(t2-t1)+"@"+(t1-t0)+","+String.valueOf(rtnuif.uni));
}
}catch(IOException e){
rtnuif.e=e;
}
return rtnuif;
}
};
submit.submit(task);
UnInvertedField uif=UnInvertedFieldUtils.takeUnf(submit);
uif.refCnt.incrementAndGet();
return uif;
}
}