/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
public class TestDocsAndPositions extends LuceneTestCase {
private String fieldName;
@Override
public void setUp() throws Exception {
super.setUp();
fieldName = "field" + random().nextInt();
}
/**
* Simple testcase for {@link PostingsEnum}
*/
public void testPositionsSimple() throws IOException {
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
newIndexWriterConfig(new MockAnalyzer(random())));
for (int i = 0; i < 39; i++) {
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setOmitNorms(true);
doc.add(newField(fieldName, "1 2 3 4 5 6 7 8 9 10 "
+ "1 2 3 4 5 6 7 8 9 10 " + "1 2 3 4 5 6 7 8 9 10 "
+ "1 2 3 4 5 6 7 8 9 10", customType));
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
int num = atLeast(13);
for (int i = 0; i < num; i++) {
BytesRef bytes = new BytesRef("1");
IndexReaderContext topReaderContext = reader.getContext();
for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
PostingsEnum docsAndPosEnum = getDocsAndPositions(
leafReaderContext.reader(), bytes);
assertNotNull(docsAndPosEnum);
if (leafReaderContext.reader().maxDoc() == 0) {
continue;
}
final int advance = docsAndPosEnum.advance(random().nextInt(leafReaderContext.reader().maxDoc()));
do {
String msg = "Advanced to: " + advance + " current doc: "
+ docsAndPosEnum.docID(); // TODO: + " usePayloads: " + usePayload;
assertEquals(msg, 4, docsAndPosEnum.freq());
assertEquals(msg, 0, docsAndPosEnum.nextPosition());
assertEquals(msg, 4, docsAndPosEnum.freq());
assertEquals(msg, 10, docsAndPosEnum.nextPosition());
assertEquals(msg, 4, docsAndPosEnum.freq());
assertEquals(msg, 20, docsAndPosEnum.nextPosition());
assertEquals(msg, 4, docsAndPosEnum.freq());
assertEquals(msg, 30, docsAndPosEnum.nextPosition());
} while (docsAndPosEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
}
}
reader.close();
directory.close();
}
public PostingsEnum getDocsAndPositions(LeafReader reader,
BytesRef bytes) throws IOException {
Terms terms = reader.terms(fieldName);
if (terms != null) {
TermsEnum te = terms.iterator();
if (te.seekExact(bytes)) {
return te.postings(null, PostingsEnum.ALL);
}
}
return null;
}
/**
* this test indexes random numbers within a range into a field and checks
* their occurrences by searching for a number from that range selected at
* random. All positions for that number are saved up front and compared to
* the enums positions.
*/
public void testRandomPositions() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy()));
int numDocs = atLeast(47);
int max = 1051;
int term = random().nextInt(max);
Integer[][] positionsInDoc = new Integer[numDocs][];
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setOmitNorms(true);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
ArrayList<Integer> positions = new ArrayList<>();
StringBuilder builder = new StringBuilder();
int num = atLeast(131);
for (int j = 0; j < num; j++) {
int nextInt = random().nextInt(max);
builder.append(nextInt).append(" ");
if (nextInt == term) {
positions.add(Integer.valueOf(j));
}
}
if (positions.size() == 0) {
builder.append(term);
positions.add(num);
}
doc.add(newField(fieldName, builder.toString(), customType));
positionsInDoc[i] = positions.toArray(new Integer[0]);
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
int num = atLeast(13);
for (int i = 0; i < num; i++) {
BytesRef bytes = new BytesRef("" + term);
IndexReaderContext topReaderContext = reader.getContext();
for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
PostingsEnum docsAndPosEnum = getDocsAndPositions(
leafReaderContext.reader(), bytes);
assertNotNull(docsAndPosEnum);
int initDoc = 0;
int maxDoc = leafReaderContext.reader().maxDoc();
// initially advance or do next doc
if (random().nextBoolean()) {
initDoc = docsAndPosEnum.nextDoc();
} else {
initDoc = docsAndPosEnum.advance(random().nextInt(maxDoc));
}
// now run through the scorer and check if all positions are there...
do {
int docID = docsAndPosEnum.docID();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
break;
}
Integer[] pos = positionsInDoc[leafReaderContext.docBase + docID];
assertEquals(pos.length, docsAndPosEnum.freq());
// number of positions read should be random - don't read all of them
// allways
final int howMany = random().nextInt(20) == 0 ? pos.length
- random().nextInt(pos.length) : pos.length;
for (int j = 0; j < howMany; j++) {
assertEquals("iteration: " + i + " initDoc: " + initDoc + " doc: "
+ docID + " base: " + leafReaderContext.docBase
+ " positions: " + Arrays.toString(pos) /* TODO: + " usePayloads: "
+ usePayload*/, pos[j].intValue(), docsAndPosEnum.nextPosition());
}
if (random().nextInt(10) == 0) { // once is a while advance
if (docsAndPosEnum.advance(docID + 1 + random().nextInt((maxDoc - docID))) == DocIdSetIterator.NO_MORE_DOCS) {
break;
}
}
} while (docsAndPosEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
}
}
reader.close();
dir.close();
}
public void testRandomDocs() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy()));
int numDocs = atLeast(49);
int max = 15678;
int term = random().nextInt(max);
int[] freqInDoc = new int[numDocs];
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setOmitNorms(true);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
StringBuilder builder = new StringBuilder();
for (int j = 0; j < 199; j++) {
int nextInt = random().nextInt(max);
builder.append(nextInt).append(' ');
if (nextInt == term) {
freqInDoc[i]++;
}
}
doc.add(newField(fieldName, builder.toString(), customType));
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
int num = atLeast(13);
for (int i = 0; i < num; i++) {
BytesRef bytes = new BytesRef("" + term);
IndexReaderContext topReaderContext = reader.getContext();
for (LeafReaderContext context : topReaderContext.leaves()) {
int maxDoc = context.reader().maxDoc();
PostingsEnum postingsEnum = TestUtil.docs(random(), context.reader(), fieldName, bytes, null, PostingsEnum.FREQS);
if (findNext(freqInDoc, context.docBase, context.docBase + maxDoc) == Integer.MAX_VALUE) {
assertNull(postingsEnum);
continue;
}
assertNotNull(postingsEnum);
postingsEnum.nextDoc();
for (int j = 0; j < maxDoc; j++) {
if (freqInDoc[context.docBase + j] != 0) {
assertEquals(j, postingsEnum.docID());
assertEquals(postingsEnum.freq(), freqInDoc[context.docBase +j]);
if (i % 2 == 0 && random().nextInt(10) == 0) {
int next = findNext(freqInDoc, context.docBase+j+1, context.docBase + maxDoc) - context.docBase;
int advancedTo = postingsEnum.advance(next);
if (next >= maxDoc) {
assertEquals(DocIdSetIterator.NO_MORE_DOCS, advancedTo);
} else {
assertTrue("advanced to: " +advancedTo + " but should be <= " + next, next >= advancedTo);
}
} else {
postingsEnum.nextDoc();
}
}
}
assertEquals("docBase: " + context.docBase + " maxDoc: " + maxDoc + " " + postingsEnum.getClass(), DocIdSetIterator.NO_MORE_DOCS, postingsEnum.docID());
}
}
reader.close();
dir.close();
}
private static int findNext(int[] docs, int pos, int max) {
for (int i = pos; i < max; i++) {
if( docs[i] != 0) {
return i;
}
}
return Integer.MAX_VALUE;
}
/**
* tests retrieval of positions for terms that have a large number of
* occurrences to force test of buffer refill during positions iteration.
*/
public void testLargeNumberOfPositions() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(new MockAnalyzer(random())));
int howMany = 1000;
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setOmitNorms(true);
for (int i = 0; i < 39; i++) {
Document doc = new Document();
StringBuilder builder = new StringBuilder();
for (int j = 0; j < howMany; j++) {
if (j % 2 == 0) {
builder.append("even ");
} else {
builder.append("odd ");
}
}
doc.add(newField(fieldName, builder.toString(), customType));
writer.addDocument(doc);
}
// now do searches
IndexReader reader = writer.getReader();
writer.close();
int num = atLeast(13);
for (int i = 0; i < num; i++) {
BytesRef bytes = new BytesRef("even");
IndexReaderContext topReaderContext = reader.getContext();
for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
PostingsEnum docsAndPosEnum = getDocsAndPositions(
leafReaderContext.reader(), bytes);
assertNotNull(docsAndPosEnum);
int initDoc = 0;
int maxDoc = leafReaderContext.reader().maxDoc();
// initially advance or do next doc
if (random().nextBoolean()) {
initDoc = docsAndPosEnum.nextDoc();
} else {
initDoc = docsAndPosEnum.advance(random().nextInt(maxDoc));
}
String msg = "Iteration: " + i + " initDoc: " + initDoc; // TODO: + " payloads: " + usePayload;
assertEquals(howMany / 2, docsAndPosEnum.freq());
for (int j = 0; j < howMany; j += 2) {
assertEquals("position missmatch index: " + j + " with freq: "
+ docsAndPosEnum.freq() + " -- " + msg, j,
docsAndPosEnum.nextPosition());
}
}
}
reader.close();
dir.close();
}
public void testDocsEnumStart() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("foo", "bar", Field.Store.NO));
writer.addDocument(doc);
DirectoryReader reader = writer.getReader();
LeafReader r = getOnlyLeafReader(reader);
PostingsEnum disi = TestUtil.docs(random(), r, "foo", new BytesRef("bar"), null, PostingsEnum.NONE);
int docid = disi.docID();
assertEquals(-1, docid);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
// now reuse and check again
TermsEnum te = r.terms("foo").iterator();
assertTrue(te.seekExact(new BytesRef("bar")));
disi = TestUtil.docs(random(), te, disi, PostingsEnum.NONE);
docid = disi.docID();
assertEquals(-1, docid);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
writer.close();
r.close();
dir.close();
}
public void testDocsAndPositionsEnumStart() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newTextField("foo", "bar", Field.Store.NO));
writer.addDocument(doc);
DirectoryReader reader = writer.getReader();
LeafReader r = getOnlyLeafReader(reader);
PostingsEnum disi = r.postings(new Term("foo", "bar"), PostingsEnum.ALL);
int docid = disi.docID();
assertEquals(-1, docid);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
// now reuse and check again
TermsEnum te = r.terms("foo").iterator();
assertTrue(te.seekExact(new BytesRef("bar")));
disi = te.postings(disi, PostingsEnum.ALL);
docid = disi.docID();
assertEquals(-1, docid);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
writer.close();
r.close();
dir.close();
}
}