package org.apache.blur.lucene.search;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.blur.lucene.search.IterablePaging.ProgressRef;
import org.apache.blur.lucene.search.IterablePaging.TotalHitsRef;
import org.apache.blur.thrift.generated.BlurException;
import org.apache.blur.utils.BlurIterator;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntDocValuesField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.junit.Test;
/**
* Testing the paging collector.
*
*/
public class TestingPagingCollector {
private static RAMDirectory _directory;
private int length = 1324;
@Test
public void testSimpleSearchPaging() throws Exception {
IndexReader reader = getReaderFlatScore(length);
IndexSearcherCloseable searcher = getSearcher(reader);
TotalHitsRef totalHitsRef = new TotalHitsRef();
ProgressRef progressRef = new ProgressRef();
TermQuery query = new TermQuery(new Term("f1", "value"));
IterablePaging paging = new IterablePaging(new AtomicBoolean(true), searcher, query, 100, null, null, false, null,
new DeepPagingCache());
IterablePaging itPaging = paging.skipTo(90).gather(20).totalHits(totalHitsRef).progress(progressRef);
BlurIterator<ScoreDoc, BlurException> iterator = itPaging.iterator();
int position = 90;
int searches = 1;
while (iterator.hasNext()) {
ScoreDoc sd = iterator.next();
assertEquals(position, progressRef.currentHitPosition());
assertEquals(searches, progressRef.searchesPerformed());
System.out.println("time [" + progressRef.queryTime() + "] " + "total hits [" + totalHitsRef.totalHits() + "] "
+ "searches [" + progressRef.searchesPerformed() + "] " + "position [" + progressRef.currentHitPosition()
+ "] " + "doc id [" + sd.doc + "] " + "score [" + sd.score + "]");
position++;
if (position == 100) {
searches++;
}
}
}
private IndexSearcherCloseable getSearcher(IndexReader reader) {
return new IndexSearcherCloseableBase(reader, null) {
@Override
public Directory getDirectory() {
return _directory;
}
@Override
public void close() throws IOException {
// Do nothing because it's in memory test.
}
};
}
@Test
public void testSimpleSearchPagingThroughAll() throws Exception {
IndexReader reader = getReaderFlatScore(length);
IndexSearcherCloseable searcher = getSearcher(reader);
TotalHitsRef totalHitsRef = new TotalHitsRef();
ProgressRef progressRef = new ProgressRef();
long start = System.nanoTime();
int position = 0;
DeepPagingCache deepPagingCache = new DeepPagingCache(10);
OUTER: while (true) {
MatchAllDocsQuery query = new MatchAllDocsQuery();
IterablePaging paging = new IterablePaging(new AtomicBoolean(true), searcher, query, 100, null, null, false,
null, deepPagingCache);
IterablePaging itPaging = paging.skipTo(position).totalHits(totalHitsRef).progress(progressRef);
BlurIterator<ScoreDoc, BlurException> iterator = itPaging.iterator();
while (iterator.hasNext()) {
ScoreDoc sd = iterator.next();
assertEquals(position, progressRef.currentHitPosition());
System.out.println("time [" + progressRef.queryTime() + "] " + "total hits [" + totalHitsRef.totalHits() + "] "
+ "searches [" + progressRef.searchesPerformed() + "] " + "position [" + progressRef.currentHitPosition()
+ "] " + "doc id [" + sd.doc + "] " + "score [" + sd.score + "]");
position++;
if (position % 100 == 0) {
continue OUTER;
}
}
break OUTER;
}
long end = System.nanoTime();
assertEquals(length, position);
System.out.println("Took [" + (end - start) / 1000000.0 + " ms]");
}
@Test
public void testSimpleSearchPagingWithSorting() throws Exception {
IndexReader reader = getReaderFlatScore(length);
IndexSearcherCloseable searcher = getSearcher(reader);
TotalHitsRef totalHitsRef = new TotalHitsRef();
ProgressRef progressRef = new ProgressRef();
printHeapSize();
TermQuery query = new TermQuery(new Term("f1", "value"));
Sort sort = new Sort(new SortField("index", Type.INT, true));
IterablePaging paging = new IterablePaging(new AtomicBoolean(true), searcher, query, 100, null, null, false, sort,
new DeepPagingCache());
IterablePaging itPaging = paging.skipTo(90).gather(20).totalHits(totalHitsRef).progress(progressRef);
BlurIterator<ScoreDoc, BlurException> iterator = itPaging.iterator();
int position = 90;
int searches = 1;
while (iterator.hasNext()) {
ScoreDoc sd = iterator.next();
assertEquals(position, progressRef.currentHitPosition());
assertEquals(searches, progressRef.searchesPerformed());
System.out.println("time [" + progressRef.queryTime() + "] " + "total hits [" + totalHitsRef.totalHits() + "] "
+ "searches [" + progressRef.searchesPerformed() + "] " + "position [" + progressRef.currentHitPosition()
+ "] " + "doc id [" + sd.doc + "] " + "score [" + sd.score + "]");
position++;
if (position == 100) {
searches++;
}
}
printHeapSize();
}
private void printHeapSize() {
System.gc();
System.gc();
System.out.println("heap size=" + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed());
}
private static IndexReader getReaderFlatScore(int length) throws Exception {
_directory = new RAMDirectory();
IndexWriter indexWriter = new IndexWriter(_directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()));
for (int i = 0; i < length; i++) {
Document document = new Document();
document.add(new StringField("f1", "value", Store.NO));
document.add(new IntDocValuesField("index", i));
document.add(new IntField("index", i, Store.YES));
indexWriter.addDocument(document);
}
indexWriter.close();
return DirectoryReader.open(_directory);
}
static byte[] toBytes(int val) {
byte[] b = new byte[4];
b[3] = (byte) (val);
b[2] = (byte) (val >>> 8);
b[1] = (byte) (val >>> 16);
b[0] = (byte) (val >>> 24);
return b;
}
static int toInt(byte[] b) {
return ((b[3] & 0xFF)) + ((b[2] & 0xFF) << 8) + ((b[1] & 0xFF) << 16) + ((b[0]) << 24);
}
}