package org.apache.lucene.index;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
public class TestIndexReaderReopen extends LuceneTestCase {
private File indexDir;
public void testReopen() throws Exception {
Random random = newRandom();
final Directory dir1 = newDirectory(random);
createIndex(random, dir1, false);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir1);
}
@Override
protected IndexReader openReader() throws IOException {
return IndexReader.open(dir1, false);
}
});
dir1.close();
final Directory dir2 = newDirectory(random);
createIndex(random, dir2, true);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir2);
}
@Override
protected IndexReader openReader() throws IOException {
return IndexReader.open(dir2, false);
}
});
dir2.close();
}
public void testParallelReaderReopen() throws Exception {
Random random = newRandom();
final Directory dir1 = newDirectory(random);
createIndex(random, dir1, true);
final Directory dir2 = newDirectory(random);
createIndex(random, dir2, true);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir1);
TestIndexReaderReopen.modifyIndex(i, dir2);
}
@Override
protected IndexReader openReader() throws IOException {
ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir1, false));
pr.add(IndexReader.open(dir2, false));
return pr;
}
});
dir1.close();
dir2.close();
final Directory dir3 = newDirectory(random);
createIndex(random, dir3, true);
final Directory dir4 = newDirectory(random);
createIndex(random, dir4, true);
performTestsWithExceptionInReopen(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir3);
TestIndexReaderReopen.modifyIndex(i, dir4);
}
@Override
protected IndexReader openReader() throws IOException {
ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir3, false));
pr.add(IndexReader.open(dir4, false));
// Does not implement reopen, so
// hits exception:
pr.add(new FilterIndexReader(IndexReader.open(dir3, false)));
return pr;
}
});
dir3.close();
dir4.close();
}
// LUCENE-1228: IndexWriter.commit() does not update the index version
// populate an index in iterations.
// at the end of every iteration, commit the index and reopen/recreate the reader.
// in each iteration verify the work of previous iteration.
// try this once with reopen once recreate, on both RAMDir and FSDir.
public void testCommitReopenFS () throws IOException {
Directory dir = FSDirectory.open(indexDir);
doTestReopenWithCommit(newRandom(), dir, true);
dir.close();
}
public void testCommitRecreateFS () throws IOException {
Directory dir = FSDirectory.open(indexDir);
doTestReopenWithCommit(newRandom(), dir, false);
dir.close();
}
public void testCommitReopenRAM () throws IOException {
Random random = newRandom();
Directory dir = newDirectory(random);
doTestReopenWithCommit(random, dir, true);
dir.close();
}
public void testCommitRecreateRAM () throws IOException {
Random random = newRandom();
Directory dir = newDirectory(random);
doTestReopenWithCommit(random, dir, false);
dir.close();
}
private void doTestReopenWithCommit (Random random, Directory dir, boolean withReopen) throws IOException {
IndexWriter iwriter = new IndexWriter(dir, newIndexWriterConfig(random,
TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(
OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()));
iwriter.commit();
IndexReader reader = IndexReader.open(dir, false);
try {
int M = 3;
for (int i=0; i<4; i++) {
for (int j=0; j<M; j++) {
Document doc = new Document();
doc.add(new Field("id", i+"_"+j, Store.YES, Index.NOT_ANALYZED));
doc.add(new Field("id2", i+"_"+j, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
doc.add(new Field("id3", i+"_"+j, Store.YES, Index.NO));
iwriter.addDocument(doc);
if (i>0) {
int k = i-1;
int n = j + k*M;
Document prevItereationDoc = reader.document(n);
assertNotNull(prevItereationDoc);
String id = prevItereationDoc.get("id");
assertEquals(k+"_"+j, id);
}
}
iwriter.commit();
if (withReopen) {
// reopen
IndexReader r2 = reader.reopen();
if (reader != r2) {
reader.close();
reader = r2;
}
} else {
// recreate
reader.close();
reader = IndexReader.open(dir, false);
}
}
} finally {
iwriter.close();
reader.close();
}
}
public void testMultiReaderReopen() throws Exception {
Random random = newRandom();
final Directory dir1 = newDirectory(random);
createIndex(random, dir1, true);
final Directory dir2 = newDirectory(random);
createIndex(random, dir2, true);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir1);
TestIndexReaderReopen.modifyIndex(i, dir2);
}
@Override
protected IndexReader openReader() throws IOException {
return new MultiReader(new IndexReader[]
{IndexReader.open(dir1, false),
IndexReader.open(dir2, false)});
}
});
dir1.close();
dir2.close();
final Directory dir3 = newDirectory(random);
createIndex(random, dir3, true);
final Directory dir4 = newDirectory(random);
createIndex(random, dir4, true);
performTestsWithExceptionInReopen(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir3);
TestIndexReaderReopen.modifyIndex(i, dir4);
}
@Override
protected IndexReader openReader() throws IOException {
return new MultiReader(new IndexReader[]
{IndexReader.open(dir3, false),
IndexReader.open(dir4, false),
// Does not implement reopen, so
// hits exception:
new FilterIndexReader(IndexReader.open(dir3, false))});
}
});
dir3.close();
dir4.close();
}
public void testMixedReaders() throws Exception {
Random random = newRandom();
final Directory dir1 = newDirectory(random);
createIndex(random, dir1, true);
final Directory dir2 = newDirectory(random);
createIndex(random, dir2, true);
final Directory dir3 = newDirectory(random);
createIndex(random, dir3, false);
final Directory dir4 = newDirectory(random);
createIndex(random, dir4, true);
final Directory dir5 = newDirectory(random);
createIndex(random, dir5, false);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
// only change norms in this index to maintain the same number of docs for each of ParallelReader's subreaders
if (i == 1) TestIndexReaderReopen.modifyIndex(i, dir1);
TestIndexReaderReopen.modifyIndex(i, dir4);
TestIndexReaderReopen.modifyIndex(i, dir5);
}
@Override
protected IndexReader openReader() throws IOException {
ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir1, false));
pr.add(IndexReader.open(dir2, false));
MultiReader mr = new MultiReader(new IndexReader[] {
IndexReader.open(dir3, false), IndexReader.open(dir4, false)});
return new MultiReader(new IndexReader[] {
pr, mr, IndexReader.open(dir5, false)});
}
});
dir1.close();
dir2.close();
dir3.close();
dir4.close();
dir5.close();
}
private void performDefaultTests(TestReopen test) throws Exception {
IndexReader index1 = test.openReader();
IndexReader index2 = test.openReader();
TestIndexReader.assertIndexEquals(index1, index2);
// verify that reopen() does not return a new reader instance
// in case the index has no changes
ReaderCouple couple = refreshReader(index2, false);
assertTrue(couple.refreshedReader == index2);
couple = refreshReader(index2, test, 0, true);
index1.close();
index1 = couple.newReader;
IndexReader index2_refreshed = couple.refreshedReader;
index2.close();
// test if refreshed reader and newly opened reader return equal results
TestIndexReader.assertIndexEquals(index1, index2_refreshed);
index2_refreshed.close();
assertReaderClosed(index2, true, true);
assertReaderClosed(index2_refreshed, true, true);
index2 = test.openReader();
for (int i = 1; i < 4; i++) {
index1.close();
couple = refreshReader(index2, test, i, true);
// refresh IndexReader
index2.close();
index2 = couple.refreshedReader;
index1 = couple.newReader;
TestIndexReader.assertIndexEquals(index1, index2);
}
index1.close();
index2.close();
assertReaderClosed(index1, true, true);
assertReaderClosed(index2, true, true);
}
public void testReferenceCounting() throws IOException {
Random random = newRandom();
for (int mode = 0; mode < 4; mode++) {
Directory dir1 = newDirectory(random);
createIndex(random, dir1, true);
IndexReader reader0 = IndexReader.open(dir1, false);
assertRefCountEquals(1, reader0);
assertTrue(reader0 instanceof DirectoryReader);
IndexReader[] subReaders0 = reader0.getSequentialSubReaders();
for (int i = 0; i < subReaders0.length; i++) {
assertRefCountEquals(1, subReaders0[i]);
}
// delete first document, so that only one of the subReaders have to be re-opened
IndexReader modifier = IndexReader.open(dir1, false);
modifier.deleteDocument(0);
modifier.close();
IndexReader reader1 = refreshReader(reader0, true).refreshedReader;
assertTrue(reader1 instanceof DirectoryReader);
IndexReader[] subReaders1 = reader1.getSequentialSubReaders();
assertEquals(subReaders0.length, subReaders1.length);
for (int i = 0; i < subReaders0.length; i++) {
if (subReaders0[i] != subReaders1[i]) {
assertRefCountEquals(1, subReaders0[i]);
assertRefCountEquals(1, subReaders1[i]);
} else {
assertRefCountEquals(2, subReaders0[i]);
}
}
// delete first document, so that only one of the subReaders have to be re-opened
modifier = IndexReader.open(dir1, false);
modifier.deleteDocument(1);
modifier.close();
IndexReader reader2 = refreshReader(reader1, true).refreshedReader;
assertTrue(reader2 instanceof DirectoryReader);
IndexReader[] subReaders2 = reader2.getSequentialSubReaders();
assertEquals(subReaders1.length, subReaders2.length);
for (int i = 0; i < subReaders2.length; i++) {
if (subReaders2[i] == subReaders1[i]) {
if (subReaders1[i] == subReaders0[i]) {
assertRefCountEquals(3, subReaders2[i]);
} else {
assertRefCountEquals(2, subReaders2[i]);
}
} else {
assertRefCountEquals(1, subReaders2[i]);
if (subReaders0[i] == subReaders1[i]) {
assertRefCountEquals(2, subReaders2[i]);
assertRefCountEquals(2, subReaders0[i]);
} else {
assertRefCountEquals(1, subReaders0[i]);
assertRefCountEquals(1, subReaders1[i]);
}
}
}
IndexReader reader3 = refreshReader(reader0, true).refreshedReader;
assertTrue(reader3 instanceof DirectoryReader);
IndexReader[] subReaders3 = reader3.getSequentialSubReaders();
assertEquals(subReaders3.length, subReaders0.length);
// try some permutations
switch (mode) {
case 0:
reader0.close();
reader1.close();
reader2.close();
reader3.close();
break;
case 1:
reader3.close();
reader2.close();
reader1.close();
reader0.close();
break;
case 2:
reader2.close();
reader3.close();
reader0.close();
reader1.close();
break;
case 3:
reader1.close();
reader3.close();
reader2.close();
reader0.close();
break;
}
assertReaderClosed(reader0, true, true);
assertReaderClosed(reader1, true, true);
assertReaderClosed(reader2, true, true);
assertReaderClosed(reader3, true, true);
dir1.close();
}
}
public void testReferenceCountingMultiReader() throws IOException {
Random random = newRandom();
for (int mode = 0; mode <=1; mode++) {
Directory dir1 = newDirectory(random);
createIndex(random, dir1, false);
Directory dir2 = newDirectory(random);
createIndex(random, dir2, true);
IndexReader reader1 = IndexReader.open(dir1, false);
assertRefCountEquals(1, reader1);
IndexReader initReader2 = IndexReader.open(dir2, false);
IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, initReader2}, (mode == 0));
modifyIndex(0, dir2);
assertRefCountEquals(1 + mode, reader1);
IndexReader multiReader2 = multiReader1.reopen();
// index1 hasn't changed, so multiReader2 should share reader1 now with multiReader1
assertRefCountEquals(2 + mode, reader1);
modifyIndex(0, dir1);
IndexReader reader2 = reader1.reopen();
assertRefCountEquals(2 + mode, reader1);
if (mode == 1) {
initReader2.close();
}
modifyIndex(1, dir1);
IndexReader reader3 = reader2.reopen();
assertRefCountEquals(2 + mode, reader1);
assertRefCountEquals(1, reader2);
multiReader1.close();
assertRefCountEquals(1 + mode, reader1);
multiReader1.close();
assertRefCountEquals(1 + mode, reader1);
if (mode == 1) {
initReader2.close();
}
reader1.close();
assertRefCountEquals(1, reader1);
multiReader2.close();
assertRefCountEquals(0, reader1);
multiReader2.close();
assertRefCountEquals(0, reader1);
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, false);
reader2.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, false);
reader2.close();
assertRefCountEquals(0, reader1);
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, true);
dir1.close();
dir2.close();
}
}
public void testReferenceCountingParallelReader() throws IOException {
Random random = newRandom();
for (int mode = 0; mode <=1; mode++) {
Directory dir1 = newDirectory(random);
createIndex(random, dir1, false);
Directory dir2 = newDirectory(random);
createIndex(random, dir2, true);
IndexReader reader1 = IndexReader.open(dir1, false);
assertRefCountEquals(1, reader1);
ParallelReader parallelReader1 = new ParallelReader(mode == 0);
parallelReader1.add(reader1);
IndexReader initReader2 = IndexReader.open(dir2, false);
parallelReader1.add(initReader2);
modifyIndex(1, dir2);
assertRefCountEquals(1 + mode, reader1);
IndexReader parallelReader2 = parallelReader1.reopen();
// index1 hasn't changed, so parallelReader2 should share reader1 now with multiReader1
assertRefCountEquals(2 + mode, reader1);
modifyIndex(0, dir1);
modifyIndex(0, dir2);
IndexReader reader2 = reader1.reopen();
assertRefCountEquals(2 + mode, reader1);
if (mode == 1) {
initReader2.close();
}
modifyIndex(4, dir1);
IndexReader reader3 = reader2.reopen();
assertRefCountEquals(2 + mode, reader1);
assertRefCountEquals(1, reader2);
parallelReader1.close();
assertRefCountEquals(1 + mode, reader1);
parallelReader1.close();
assertRefCountEquals(1 + mode, reader1);
if (mode == 1) {
initReader2.close();
}
reader1.close();
assertRefCountEquals(1, reader1);
parallelReader2.close();
assertRefCountEquals(0, reader1);
parallelReader2.close();
assertRefCountEquals(0, reader1);
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, false);
reader2.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, false);
reader2.close();
assertRefCountEquals(0, reader1);
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, true);
dir1.close();
dir2.close();
}
}
public void testNormsRefCounting() throws IOException {
Random random = newRandom();
Directory dir1 = newDirectory(random);
createIndex(random, dir1, false);
IndexReader reader1 = IndexReader.open(dir1, false);
SegmentReader segmentReader1 = SegmentReader.getOnlySegmentReader(reader1);
IndexReader modifier = IndexReader.open(dir1, false);
modifier.deleteDocument(0);
modifier.close();
IndexReader reader2 = reader1.reopen();
modifier = IndexReader.open(dir1, false);
modifier.setNorm(1, "field1", 50);
modifier.setNorm(1, "field2", 50);
modifier.close();
IndexReader reader3 = reader2.reopen();
SegmentReader segmentReader3 = SegmentReader.getOnlySegmentReader(reader3);
modifier = IndexReader.open(dir1, false);
modifier.deleteDocument(2);
modifier.close();
IndexReader reader4 = reader3.reopen();
modifier = IndexReader.open(dir1, false);
modifier.deleteDocument(3);
modifier.close();
IndexReader reader5 = reader3.reopen();
// Now reader2-reader5 references reader1. reader1 and reader2
// share the same norms. reader3, reader4, reader5 also share norms.
assertRefCountEquals(1, reader1);
assertFalse(segmentReader1.normsClosed());
reader1.close();
assertRefCountEquals(0, reader1);
assertFalse(segmentReader1.normsClosed());
reader2.close();
assertRefCountEquals(0, reader1);
// now the norms for field1 and field2 should be closed
assertTrue(segmentReader1.normsClosed("field1"));
assertTrue(segmentReader1.normsClosed("field2"));
// but the norms for field3 and field4 should still be open
assertFalse(segmentReader1.normsClosed("field3"));
assertFalse(segmentReader1.normsClosed("field4"));
reader3.close();
assertRefCountEquals(0, reader1);
assertFalse(segmentReader3.normsClosed());
reader5.close();
assertRefCountEquals(0, reader1);
assertFalse(segmentReader3.normsClosed());
reader4.close();
assertRefCountEquals(0, reader1);
// and now all norms that reader1 used should be closed
assertTrue(segmentReader1.normsClosed());
// now that reader3, reader4 and reader5 are closed,
// the norms that those three readers shared should be
// closed as well
assertTrue(segmentReader3.normsClosed());
dir1.close();
}
private void performTestsWithExceptionInReopen(TestReopen test) throws Exception {
IndexReader index1 = test.openReader();
IndexReader index2 = test.openReader();
TestIndexReader.assertIndexEquals(index1, index2);
try {
refreshReader(index1, test, 0, true);
fail("Expected exception not thrown.");
} catch (Exception e) {
// expected exception
}
// index2 should still be usable and unaffected by the failed reopen() call
TestIndexReader.assertIndexEquals(index1, index2);
index1.close();
index2.close();
}
public void testThreadSafety() throws Exception {
Random random = newRandom();
final Directory dir = newDirectory(random);
final int n = 30 * RANDOM_MULTIPLIER;
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random,
TEST_VERSION_CURRENT, new MockAnalyzer()));
for (int i = 0; i < n; i++) {
writer.addDocument(createDocument(i, 3));
}
writer.optimize();
writer.close();
final TestReopen test = new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
if (i % 3 == 0) {
IndexReader modifier = IndexReader.open(dir, false);
modifier.setNorm(i, "field1", 50);
modifier.close();
} else if (i % 3 == 1) {
IndexReader modifier = IndexReader.open(dir, false);
modifier.deleteDocument(i % modifier.maxDoc());
modifier.close();
} else {
IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()));
modifier.addDocument(createDocument(n + i, 6));
modifier.close();
}
}
@Override
protected IndexReader openReader() throws IOException {
return IndexReader.open(dir, false);
}
};
final List<ReaderCouple> readers = Collections.synchronizedList(new ArrayList<ReaderCouple>());
IndexReader firstReader = IndexReader.open(dir, false);
IndexReader reader = firstReader;
final Random rnd = random;
ReaderThread[] threads = new ReaderThread[n];
final Set<IndexReader> readersToClose = Collections.synchronizedSet(new HashSet<IndexReader>());
for (int i = 0; i < n; i++) {
if (i % 2 == 0) {
IndexReader refreshed = reader.reopen();
if (refreshed != reader) {
readersToClose.add(reader);
}
reader = refreshed;
}
final IndexReader r = reader;
final int index = i;
ReaderThreadTask task;
if (i < 4 ||( i >=10 && i < 14) || i > 18) {
task = new ReaderThreadTask() {
@Override
public void run() throws Exception {
while (!stopped) {
if (index % 2 == 0) {
// refresh reader synchronized
ReaderCouple c = (refreshReader(r, test, index, true));
readersToClose.add(c.newReader);
readersToClose.add(c.refreshedReader);
readers.add(c);
// prevent too many readers
break;
} else {
// not synchronized
IndexReader refreshed = r.reopen();
IndexSearcher searcher = new IndexSearcher(refreshed);
ScoreDoc[] hits = searcher.search(
new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))),
null, 1000).scoreDocs;
if (hits.length > 0) {
searcher.doc(hits[0].doc);
}
// r might have changed because this is not a
// synchronized method. However we don't want
// to make it synchronized to test
// thread-safety of IndexReader.close().
// That's why we add refreshed also to
// readersToClose, because double closing is fine
if (refreshed != r) {
refreshed.close();
}
readersToClose.add(refreshed);
}
synchronized(this) {
wait(1000);
}
}
}
};
} else {
task = new ReaderThreadTask() {
@Override
public void run() throws Exception {
while (!stopped) {
int numReaders = readers.size();
if (numReaders > 0) {
ReaderCouple c = readers.get(rnd.nextInt(numReaders));
TestIndexReader.assertIndexEquals(c.newReader, c.refreshedReader);
}
synchronized(this) {
wait(100);
}
}
}
};
}
threads[i] = new ReaderThread(task);
threads[i].start();
}
synchronized(this) {
wait(1000);
}
for (int i = 0; i < n; i++) {
if (threads[i] != null) {
threads[i].stopThread();
}
}
for (int i = 0; i < n; i++) {
if (threads[i] != null) {
threads[i].join();
if (threads[i].error != null) {
String msg = "Error occurred in thread " + threads[i].getName() + ":\n" + threads[i].error.getMessage();
fail(msg);
}
}
}
for (final IndexReader readerToClose : readersToClose) {
readerToClose.close();
}
firstReader.close();
reader.close();
for (final IndexReader readerToClose : readersToClose) {
assertReaderClosed(readerToClose, true, true);
}
assertReaderClosed(reader, true, true);
assertReaderClosed(firstReader, true, true);
dir.close();
}
private static class ReaderCouple {
ReaderCouple(IndexReader r1, IndexReader r2) {
newReader = r1;
refreshedReader = r2;
}
IndexReader newReader;
IndexReader refreshedReader;
}
private abstract static class ReaderThreadTask {
protected boolean stopped;
public void stop() {
this.stopped = true;
}
public abstract void run() throws Exception;
}
private static class ReaderThread extends Thread {
private ReaderThreadTask task;
private Throwable error;
ReaderThread(ReaderThreadTask task) {
this.task = task;
}
public void stopThread() {
this.task.stop();
}
@Override
public void run() {
try {
this.task.run();
} catch (Throwable r) {
r.printStackTrace(System.out);
this.error = r;
}
}
}
private Object createReaderMutex = new Object();
private ReaderCouple refreshReader(IndexReader reader, boolean hasChanges) throws IOException {
return refreshReader(reader, null, -1, hasChanges);
}
ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException {
synchronized (createReaderMutex) {
IndexReader r = null;
if (test != null) {
test.modifyIndex(modify);
r = test.openReader();
}
IndexReader refreshed = null;
try {
refreshed = reader.reopen();
} finally {
if (refreshed == null && r != null) {
// Hit exception -- close opened reader
r.close();
}
}
if (hasChanges) {
if (refreshed == reader) {
fail("No new IndexReader instance created during refresh.");
}
} else {
if (refreshed != reader) {
fail("New IndexReader instance created during refresh even though index had no changes.");
}
}
return new ReaderCouple(r, refreshed);
}
}
public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException {
IndexWriter.unlock(dir);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(random,
TEST_VERSION_CURRENT, new MockAnalyzer())
.setMergePolicy(new LogDocMergePolicy()));
for (int i = 0; i < 100; i++) {
w.addDocument(createDocument(i, 4));
if (multiSegment && (i % 10) == 0) {
w.commit();
}
}
if (!multiSegment) {
w.optimize();
}
w.close();
IndexReader r = IndexReader.open(dir, false);
if (multiSegment) {
assertTrue(r.getSequentialSubReaders().length > 1);
} else {
assertTrue(r.getSequentialSubReaders().length == 1);
}
r.close();
}
public static Document createDocument(int n, int numFields) {
StringBuilder sb = new StringBuilder();
Document doc = new Document();
sb.append("a");
sb.append(n);
doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED));
doc.add(new Field("fielda", sb.toString(), Store.YES, Index.NOT_ANALYZED_NO_NORMS));
doc.add(new Field("fieldb", sb.toString(), Store.YES, Index.NO));
sb.append(" b");
sb.append(n);
for (int i = 1; i < numFields; i++) {
doc.add(new Field("field" + (i+1), sb.toString(), Store.YES, Index.ANALYZED));
}
return doc;
}
static void modifyIndex(int i, Directory dir) throws IOException {
switch (i) {
case 0: {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
w.deleteDocuments(new Term("field2", "a11"));
w.deleteDocuments(new Term("field2", "b30"));
w.close();
break;
}
case 1: {
IndexReader reader = IndexReader.open(dir, false);
reader.setNorm(4, "field1", 123);
reader.setNorm(44, "field2", 222);
reader.setNorm(44, "field4", 22);
reader.close();
break;
}
case 2: {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
w.optimize();
w.close();
break;
}
case 3: {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
w.addDocument(createDocument(101, 4));
w.optimize();
w.addDocument(createDocument(102, 4));
w.addDocument(createDocument(103, 4));
w.close();
break;
}
case 4: {
IndexReader reader = IndexReader.open(dir, false);
reader.setNorm(5, "field1", 123);
reader.setNorm(55, "field2", 222);
reader.close();
break;
}
case 5: {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
w.addDocument(createDocument(101, 4));
w.close();
break;
}
}
}
private void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) {
assertEquals(0, reader.getRefCount());
if (checkNormsClosed && reader instanceof SegmentReader) {
assertTrue(((SegmentReader) reader).normsClosed());
}
if (checkSubReaders) {
if (reader instanceof DirectoryReader) {
IndexReader[] subReaders = reader.getSequentialSubReaders();
for (int i = 0; i < subReaders.length; i++) {
assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
}
}
if (reader instanceof MultiReader) {
IndexReader[] subReaders = reader.getSequentialSubReaders();
for (int i = 0; i < subReaders.length; i++) {
assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
}
}
if (reader instanceof ParallelReader) {
IndexReader[] subReaders = ((ParallelReader) reader).getSubReaders();
for (int i = 0; i < subReaders.length; i++) {
assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
}
}
}
}
/*
private void assertReaderOpen(IndexReader reader) {
reader.ensureOpen();
if (reader instanceof DirectoryReader) {
IndexReader[] subReaders = reader.getSequentialSubReaders();
for (int i = 0; i < subReaders.length; i++) {
assertReaderOpen(subReaders[i]);
}
}
}
*/
private void assertRefCountEquals(int refCount, IndexReader reader) {
assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount());
}
private abstract static class TestReopen {
protected abstract IndexReader openReader() throws IOException;
protected abstract void modifyIndex(int i) throws IOException;
}
@Override
protected void setUp() throws Exception {
super.setUp();
indexDir = new File(TEMP_DIR, "IndexReaderReopen");
}
public void testCloseOrig() throws Throwable {
Random random = newRandom();
Directory dir = newDirectory(random);
createIndex(random, dir, false);
IndexReader r1 = IndexReader.open(dir, false);
IndexReader r2 = IndexReader.open(dir, false);
r2.deleteDocument(0);
r2.close();
IndexReader r3 = r1.reopen();
assertTrue(r1 != r3);
r1.close();
try {
r1.document(2);
fail("did not hit exception");
} catch (AlreadyClosedException ace) {
// expected
}
r3.close();
dir.close();
}
public void testDeletes() throws Throwable {
Random random = newRandom();
Directory dir = newDirectory(random);
createIndex(random, dir, false); // Create an index with a bunch of docs (1 segment)
modifyIndex(0, dir); // Get delete bitVector on 1st segment
modifyIndex(5, dir); // Add a doc (2 segments)
IndexReader r1 = IndexReader.open(dir, false); // MSR
modifyIndex(5, dir); // Add another doc (3 segments)
IndexReader r2 = r1.reopen(); // MSR
assertTrue(r1 != r2);
SegmentReader sr1 = (SegmentReader) r1.getSequentialSubReaders()[0]; // Get SRs for the first segment from original
SegmentReader sr2 = (SegmentReader) r2.getSequentialSubReaders()[0]; // and reopened IRs
// At this point they share the same BitVector
assertTrue(sr1.deletedDocs==sr2.deletedDocs);
r2.deleteDocument(0);
// r1 should not see the delete
final Bits r1DelDocs = MultiFields.getDeletedDocs(r1);
assertFalse(r1DelDocs != null && r1DelDocs.get(0));
// Now r2 should have made a private copy of deleted docs:
assertTrue(sr1.deletedDocs!=sr2.deletedDocs);
r1.close();
r2.close();
dir.close();
}
public void testDeletes2() throws Throwable {
Random random = newRandom();
Directory dir = newDirectory(random);
createIndex(random, dir, false);
// Get delete bitVector
modifyIndex(0, dir);
IndexReader r1 = IndexReader.open(dir, false);
// Add doc:
modifyIndex(5, dir);
IndexReader r2 = r1.reopen();
assertTrue(r1 != r2);
IndexReader[] rs2 = r2.getSequentialSubReaders();
SegmentReader sr1 = SegmentReader.getOnlySegmentReader(r1);
SegmentReader sr2 = (SegmentReader) rs2[0];
// At this point they share the same BitVector
assertTrue(sr1.deletedDocs==sr2.deletedDocs);
final BitVector delDocs = sr1.deletedDocs;
r1.close();
r2.deleteDocument(0);
assertTrue(delDocs==sr2.deletedDocs);
r2.close();
dir.close();
}
private static class KeepAllCommits implements IndexDeletionPolicy {
public void onInit(List<? extends IndexCommit> commits) {
}
public void onCommit(List<? extends IndexCommit> commits) {
}
}
public void testReopenOnCommit() throws Throwable {
Random random = newRandom();
Directory dir = newDirectory(random);
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random,
TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy(new KeepAllCommits()).setMaxBufferedDocs(-1));
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10);
for(int i=0;i<4;i++) {
Document doc = new Document();
doc.add(new Field("id", ""+i, Field.Store.NO, Field.Index.NOT_ANALYZED));
writer.addDocument(doc);
Map<String,String> data = new HashMap<String,String>();
data.put("index", i+"");
writer.commit(data);
}
for(int i=0;i<4;i++) {
writer.deleteDocuments(new Term("id", ""+i));
Map<String,String> data = new HashMap<String,String>();
data.put("index", (4+i)+"");
writer.commit(data);
}
writer.close();
IndexReader r = IndexReader.open(dir, false);
assertEquals(0, r.numDocs());
assertEquals(4, r.maxDoc());
Collection<IndexCommit> commits = IndexReader.listCommits(dir);
for (final IndexCommit commit : commits) {
IndexReader r2 = r.reopen(commit);
assertTrue(r2 != r);
// Reader should be readOnly
try {
r2.deleteDocument(0);
fail("no exception hit");
} catch (UnsupportedOperationException uoe) {
// expected
}
final Map<String,String> s = commit.getUserData();
final int v;
if (s.size() == 0) {
// First commit created by IW
v = -1;
} else {
v = Integer.parseInt(s.get("index"));
}
if (v < 4) {
assertEquals(1+v, r2.numDocs());
} else {
assertEquals(7-v, r2.numDocs());
}
r.close();
r = r2;
}
r.close();
dir.close();
}
}