/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilterDirectoryReader;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.TestUtil;
@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
boolean warmCalled;
private SearcherLifetimeManager.Pruner pruner;
public void testSearcherManager() throws Exception {
pruner = new SearcherLifetimeManager.PruneByAge(TEST_NIGHTLY ? TestUtil.nextInt(random(), 1, 20) : 1);
runTest("TestSearcherManager");
}
@Override
protected IndexSearcher getFinalSearcher() throws Exception {
if (!isNRT) {
writer.commit();
}
assertTrue(mgr.maybeRefresh() || mgr.isSearcherCurrent());
return mgr.acquire();
}
private SearcherManager mgr;
private SearcherLifetimeManager lifetimeMGR;
private final List<Long> pastSearchers = new ArrayList<>();
private boolean isNRT;
@Override
protected void doAfterWriter(final ExecutorService es) throws Exception {
final SearcherFactory factory = new SearcherFactory() {
@Override
public IndexSearcher newSearcher(IndexReader r, IndexReader previous) throws IOException {
IndexSearcher s = new IndexSearcher(r, es);
TestSearcherManager.this.warmCalled = true;
s.search(new TermQuery(new Term("body", "united")), 10);
return s;
}
};
if (random().nextBoolean()) {
// TODO: can we randomize the applyAllDeletes? But
// somehow for final searcher we must apply
// deletes...
mgr = new SearcherManager(writer, factory);
isNRT = true;
} else {
// SearcherManager needs to see empty commit:
writer.commit();
mgr = new SearcherManager(dir, factory);
isNRT = false;
assertMergedSegmentsWarmed = false;
}
lifetimeMGR = new SearcherLifetimeManager();
}
@Override
protected void doSearching(ExecutorService es, final long stopTime) throws Exception {
Thread reopenThread = new Thread() {
@Override
public void run() {
try {
if (VERBOSE) {
System.out.println("[" + Thread.currentThread().getName() + "]: launch reopen thread");
}
while(System.currentTimeMillis() < stopTime) {
Thread.sleep(TestUtil.nextInt(random(), 1, 100));
writer.commit();
Thread.sleep(TestUtil.nextInt(random(), 1, 5));
boolean block = random().nextBoolean();
if (block) {
mgr.maybeRefreshBlocking();
lifetimeMGR.prune(pruner);
} else if (mgr.maybeRefresh()) {
lifetimeMGR.prune(pruner);
}
}
} catch (Throwable t) {
if (VERBOSE) {
System.out.println("TEST: reopen thread hit exc");
t.printStackTrace(System.out);
}
failed.set(true);
throw new RuntimeException(t);
}
}
};
reopenThread.setDaemon(true);
reopenThread.start();
runSearchThreads(stopTime);
reopenThread.join();
}
@Override
protected IndexSearcher getCurrentSearcher() throws Exception {
if (random().nextInt(10) == 7) {
// NOTE: not best practice to call maybeRefresh
// synchronous to your search threads, but still we
// test as apps will presumably do this for
// simplicity:
if (mgr.maybeRefresh()) {
lifetimeMGR.prune(pruner);
}
}
IndexSearcher s = null;
synchronized(pastSearchers) {
while (pastSearchers.size() != 0 && random().nextDouble() < 0.25) {
// 1/4 of the time pull an old searcher, ie, simulate
// a user doing a follow-on action on a previous
// search (drilling down/up, clicking next/prev page,
// etc.)
final Long token = pastSearchers.get(random().nextInt(pastSearchers.size()));
s = lifetimeMGR.acquire(token);
if (s == null) {
// Searcher was pruned
pastSearchers.remove(token);
} else {
break;
}
}
}
if (s == null) {
s = mgr.acquire();
if (s.getIndexReader().numDocs() != 0) {
Long token = lifetimeMGR.record(s);
synchronized(pastSearchers) {
if (!pastSearchers.contains(token)) {
pastSearchers.add(token);
}
}
}
}
return s;
}
@Override
protected void releaseSearcher(IndexSearcher s) throws Exception {
s.getIndexReader().decRef();
}
@Override
protected void doClose() throws Exception {
assertTrue(warmCalled);
if (VERBOSE) {
System.out.println("TEST: now close SearcherManager");
}
mgr.close();
lifetimeMGR.close();
}
public void testIntermediateClose() throws IOException, InterruptedException {
Directory dir = newDirectory();
// Test can deadlock if we use SMS:
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
new MockAnalyzer(random())).setMergeScheduler(new ConcurrentMergeScheduler()));
writer.addDocument(new Document());
writer.commit();
final CountDownLatch awaitEnterWarm = new CountDownLatch(1);
final CountDownLatch awaitClose = new CountDownLatch(1);
final AtomicBoolean triedReopen = new AtomicBoolean(false);
final ExecutorService es = random().nextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
final SearcherFactory factory = new SearcherFactory() {
@Override
public IndexSearcher newSearcher(IndexReader r, IndexReader previous) {
try {
if (triedReopen.get()) {
awaitEnterWarm.countDown();
awaitClose.await();
}
} catch (InterruptedException e) {
//
}
return new IndexSearcher(r, es);
}
};
final SearcherManager searcherManager = random().nextBoolean()
? new SearcherManager(dir, factory)
: new SearcherManager(writer, random().nextBoolean(), false, factory);
if (VERBOSE) {
System.out.println("sm created");
}
IndexSearcher searcher = searcherManager.acquire();
try {
assertEquals(1, searcher.getIndexReader().numDocs());
} finally {
searcherManager.release(searcher);
}
writer.addDocument(new Document());
writer.commit();
final AtomicBoolean success = new AtomicBoolean(false);
final Throwable[] exc = new Throwable[1];
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
triedReopen.set(true);
if (VERBOSE) {
System.out.println("NOW call maybeRefresh");
}
searcherManager.maybeRefresh();
success.set(true);
} catch (AlreadyClosedException e) {
// expected
} catch (Throwable e) {
if (VERBOSE) {
System.out.println("FAIL: unexpected exc");
e.printStackTrace(System.out);
}
exc[0] = e;
// use success as the barrier here to make sure we see the write
success.set(false);
}
}
});
thread.start();
if (VERBOSE) {
System.out.println("THREAD started");
}
awaitEnterWarm.await();
if (VERBOSE) {
System.out.println("NOW call close");
}
searcherManager.close();
awaitClose.countDown();
thread.join();
expectThrows(AlreadyClosedException.class, () -> {
searcherManager.acquire();
});
assertFalse(success.get());
assertTrue(triedReopen.get());
assertNull("" + exc[0], exc[0]);
writer.close();
dir.close();
if (es != null) {
es.shutdown();
es.awaitTermination(1, TimeUnit.SECONDS);
}
}
public void testCloseTwice() throws Exception {
// test that we can close SM twice (per Closeable's contract).
Directory dir = newDirectory();
new IndexWriter(dir, new IndexWriterConfig(null)).close();
SearcherManager sm = new SearcherManager(dir, null);
sm.close();
sm.close();
dir.close();
}
public void testReferenceDecrementIllegally() throws Exception {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
new MockAnalyzer(random())).setMergeScheduler(new ConcurrentMergeScheduler()));
SearcherManager sm = new SearcherManager(writer, false, false, new SearcherFactory());
writer.addDocument(new Document());
writer.commit();
sm.maybeRefreshBlocking();
IndexSearcher acquire = sm.acquire();
IndexSearcher acquire2 = sm.acquire();
sm.release(acquire);
sm.release(acquire2);
acquire = sm.acquire();
acquire.getIndexReader().decRef();
sm.release(acquire);
expectThrows(IllegalStateException.class, () -> {
sm.acquire();
});
// sm.close(); -- already closed
writer.close();
dir.close();
}
public void testEnsureOpen() throws Exception {
Directory dir = newDirectory();
new IndexWriter(dir, new IndexWriterConfig(null)).close();
SearcherManager sm = new SearcherManager(dir, null);
IndexSearcher s = sm.acquire();
sm.close();
// this should succeed;
sm.release(s);
// this should fail
expectThrows(AlreadyClosedException.class, () -> {
sm.acquire();
});
// this should fail
expectThrows(AlreadyClosedException.class, () -> {
sm.maybeRefresh();
});
dir.close();
}
public void testListenerCalled() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));
final AtomicBoolean afterRefreshCalled = new AtomicBoolean(false);
SearcherManager sm = new SearcherManager(iw, false, false, new SearcherFactory());
sm.addListener(new ReferenceManager.RefreshListener() {
@Override
public void beforeRefresh() {
}
@Override
public void afterRefresh(boolean didRefresh) {
if (didRefresh) {
afterRefreshCalled.set(true);
}
}
});
iw.addDocument(new Document());
iw.commit();
assertFalse(afterRefreshCalled.get());
sm.maybeRefreshBlocking();
assertTrue(afterRefreshCalled.get());
sm.close();
iw.close();
dir.close();
}
public void testEvilSearcherFactory() throws Exception {
final Random random = random();
final Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random, dir);
w.commit();
final IndexReader other = DirectoryReader.open(dir);
final SearcherFactory theEvilOne = new SearcherFactory() {
@Override
public IndexSearcher newSearcher(IndexReader ignored, IndexReader previous) {
return LuceneTestCase.newSearcher(other);
}
};
expectThrows(IllegalStateException.class, () -> {
new SearcherManager(dir, theEvilOne);
});
expectThrows(IllegalStateException.class, () -> {
new SearcherManager(w.w, random.nextBoolean(), false, theEvilOne);
});
w.close();
other.close();
dir.close();
}
public void testMaybeRefreshBlockingLock() throws Exception {
// make sure that maybeRefreshBlocking releases the lock, otherwise other
// threads cannot obtain it.
final Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
w.close();
final SearcherManager sm = new SearcherManager(dir, null);
Thread t = new Thread() {
@Override
public void run() {
try {
// this used to not release the lock, preventing other threads from obtaining it.
sm.maybeRefreshBlocking();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
t.start();
t.join();
// if maybeRefreshBlocking didn't release the lock, this will fail.
assertTrue("failde to obtain the refreshLock!", sm.maybeRefresh());
sm.close();
dir.close();
}
private static class MyFilterLeafReader extends FilterLeafReader {
public MyFilterLeafReader(LeafReader in) {
super(in);
}
@Override
public CacheHelper getCoreCacheHelper() {
return in.getCoreCacheHelper();
}
@Override
public CacheHelper getReaderCacheHelper() {
return in.getReaderCacheHelper();
}
}
private static class MyFilterDirectoryReader extends FilterDirectoryReader {
public MyFilterDirectoryReader(DirectoryReader in) throws IOException {
super(in,
new FilterDirectoryReader.SubReaderWrapper() {
@Override
public LeafReader wrap(LeafReader reader) {
FilterLeafReader wrapped = new MyFilterLeafReader(reader);
assertEquals(reader, wrapped.getDelegate());
return wrapped;
}
});
}
@Override
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new MyFilterDirectoryReader(in);
}
@Override
public CacheHelper getReaderCacheHelper() {
return in.getReaderCacheHelper();
}
}
// LUCENE-6087
public void testCustomDirectoryReader() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
DirectoryReader nrtReader = w.getReader();
FilterDirectoryReader reader = new MyFilterDirectoryReader(nrtReader);
assertEquals(nrtReader, reader.getDelegate());
assertEquals(nrtReader, FilterDirectoryReader.unwrap(reader));
SearcherManager mgr = new SearcherManager(reader, null);
for(int i=0;i<10;i++) {
w.addDocument(new Document());
mgr.maybeRefresh();
IndexSearcher s = mgr.acquire();
try {
assertTrue(s.getIndexReader() instanceof MyFilterDirectoryReader);
for (LeafReaderContext ctx : s.getIndexReader().leaves()) {
assertTrue(ctx.reader() instanceof MyFilterLeafReader);
}
} finally {
mgr.release(s);
}
}
mgr.close();
w.close();
dir.close();
}
public void testPreviousReaderIsPassed() throws IOException {
final Directory dir = newDirectory();
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
w.addDocument(new Document());
class MySearcherFactory extends SearcherFactory {
IndexReader lastReader = null;
IndexReader lastPreviousReader = null;
int called = 0;
@Override
public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
called++;
lastReader = reader;
lastPreviousReader = previousReader;
return super.newSearcher(reader, previousReader);
}
}
MySearcherFactory factory = new MySearcherFactory();
final SearcherManager sm = new SearcherManager(w, random().nextBoolean(), false, factory);
assertEquals(1, factory.called);
assertNull(factory.lastPreviousReader);
assertNotNull(factory.lastReader);
IndexSearcher acquire = sm.acquire();
assertSame(factory.lastReader, acquire.getIndexReader());
sm.release(acquire);
final IndexReader lastReader = factory.lastReader;
// refresh
w.addDocument(new Document());
assertTrue(sm.maybeRefresh());
acquire = sm.acquire();
assertSame(factory.lastReader, acquire.getIndexReader());
sm.release(acquire);
assertNotNull(factory.lastPreviousReader);
assertSame(lastReader, factory.lastPreviousReader);
assertNotSame(factory.lastReader, lastReader);
assertEquals(2, factory.called);
w.close();
sm.close();
dir.close();
}
public void testConcurrentIndexCloseSearchAndRefresh() throws Exception {
final Directory dir = newFSDirectory(createTempDir());
AtomicReference<IndexWriter> writerRef = new AtomicReference<>();
final MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(IndexWriter.MAX_TERM_LENGTH);
writerRef.set(new IndexWriter(dir, newIndexWriterConfig(analyzer)));
AtomicReference<SearcherManager> mgrRef = new AtomicReference<>();
mgrRef.set(new SearcherManager(writerRef.get(), null));
final AtomicBoolean stop = new AtomicBoolean();
Thread indexThread = new Thread() {
@Override
public void run() {
try {
LineFileDocs docs = new LineFileDocs(random());
long runTimeSec = TEST_NIGHTLY ? atLeast(10) : atLeast(2);
long endTime = System.nanoTime() + runTimeSec * 1000000000;
while (System.nanoTime() < endTime) {
IndexWriter w = writerRef.get();
w.addDocument(docs.nextDoc());
if (random().nextInt(1000) == 17) {
if (random().nextBoolean()) {
w.close();
} else {
w.rollback();
}
writerRef.set(new IndexWriter(dir, newIndexWriterConfig(analyzer)));
}
}
docs.close();
stop.set(true);
if (VERBOSE) {
System.out.println("TEST: index count=" + writerRef.get().maxDoc());
}
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
};
Thread searchThread = new Thread() {
@Override
public void run() {
try {
long totCount = 0;
while (stop.get() == false) {
SearcherManager mgr = mgrRef.get();
if (mgr != null) {
IndexSearcher searcher;
try {
searcher = mgr.acquire();
} catch (AlreadyClosedException ace) {
// ok
continue;
}
totCount += searcher.getIndexReader().maxDoc();
mgr.release(searcher);
}
}
if (VERBOSE) {
System.out.println("TEST: search totCount=" + totCount);
}
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
};
Thread refreshThread = new Thread() {
@Override
public void run() {
try {
int refreshCount = 0;
int aceCount = 0;
while (stop.get() == false) {
SearcherManager mgr = mgrRef.get();
if (mgr != null) {
refreshCount++;
try {
mgr.maybeRefreshBlocking();
} catch (AlreadyClosedException ace) {
// ok
aceCount++;
continue;
}
}
}
if (VERBOSE) {
System.out.println("TEST: refresh count=" + refreshCount + " aceCount=" + aceCount);
}
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
};
Thread closeThread = new Thread() {
@Override
public void run() {
try {
int closeCount = 0;
int aceCount = 0;
while (stop.get() == false) {
SearcherManager mgr = mgrRef.get();
assert mgr != null;
mgr.close();
closeCount++;
while (stop.get() == false) {
try {
mgrRef.set(new SearcherManager(writerRef.get(), null));
break;
} catch (AlreadyClosedException ace) {
// ok
aceCount++;
}
}
}
if (VERBOSE) {
System.out.println("TEST: close count=" + closeCount + " aceCount=" + aceCount);
}
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
};
indexThread.start();
searchThread.start();
refreshThread.start();
closeThread.start();
indexThread.join();
searchThread.join();
refreshThread.join();
closeThread.join();
mgrRef.get().close();
writerRef.get().close();
dir.close();
}
}