package org.apache.lucene.index;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
public class TestAddIndexes extends LuceneTestCase {
public void testSimpleCase() throws IOException {
// main directory
Directory dir = newDirectory();
// two auxiliary directories
Directory aux = newDirectory();
Directory aux2 = newDirectory();
IndexWriter writer = null;
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random))
.setOpenMode(OpenMode.CREATE));
// add 100 documents
addDocs(writer, 100);
assertEquals(100, writer.maxDoc());
writer.close();
writer = newWriter(
aux,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.CREATE).
setMergePolicy(newLogMergePolicy(false))
);
// add 40 documents in separate files
addDocs(writer, 40);
assertEquals(40, writer.maxDoc());
writer.close();
writer = newWriter(aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE));
// add 50 documents in compound files
addDocs2(writer, 50);
assertEquals(50, writer.maxDoc());
writer.close();
// test doc count before segments are merged
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
assertEquals(100, writer.maxDoc());
writer.addIndexes(new Directory[] { aux, aux2 });
assertEquals(190, writer.maxDoc());
writer.close();
// make sure the old index is correct
verifyNumDocs(aux, 40);
// make sure the new index is correct
verifyNumDocs(dir, 190);
// now add another set in.
Directory aux3 = newDirectory();
writer = newWriter(aux3, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
// add 40 documents
addDocs(writer, 40);
assertEquals(40, writer.maxDoc());
writer.close();
// test doc count before segments are merged
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
assertEquals(190, writer.maxDoc());
writer.addIndexes(new Directory[] { aux3 });
assertEquals(230, writer.maxDoc());
writer.close();
// make sure the new index is correct
verifyNumDocs(dir, 230);
verifyTermDocs(dir, new Term("content", "aaa"), 180);
verifyTermDocs(dir, new Term("content", "bbb"), 50);
// now fully merge it.
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
writer.forceMerge(1);
writer.close();
// make sure the new index is correct
verifyNumDocs(dir, 230);
verifyTermDocs(dir, new Term("content", "aaa"), 180);
verifyTermDocs(dir, new Term("content", "bbb"), 50);
// now add a single document
Directory aux4 = newDirectory();
writer = newWriter(aux4, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
addDocs2(writer, 1);
writer.close();
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
assertEquals(230, writer.maxDoc());
writer.addIndexes(new Directory[] { aux4 });
assertEquals(231, writer.maxDoc());
writer.close();
verifyNumDocs(dir, 231);
verifyTermDocs(dir, new Term("content", "bbb"), 51);
dir.close();
aux.close();
aux2.close();
aux3.close();
aux4.close();
}
public void testWithPendingDeletes() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
setUpDirs(dir, aux);
IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
writer.setInfoStream(VERBOSE ? System.out : null);
writer.setInfoStream(VERBOSE ? System.out : null);
writer.addIndexes(aux);
// Adds 10 docs, then replaces them with another 10
// docs, so 10 pending deletes:
for (int i = 0; i < 20; i++) {
Document doc = new Document();
doc.add(newField("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(newField("content", "bbb " + i, Field.Store.NO,
Field.Index.ANALYZED));
writer.updateDocument(new Term("id", "" + (i%10)), doc);
}
// Deletes one of the 10 added docs, leaving 9:
PhraseQuery q = new PhraseQuery();
q.add(new Term("content", "bbb"));
q.add(new Term("content", "14"));
writer.deleteDocuments(q);
writer.forceMerge(1);
writer.commit();
verifyNumDocs(dir, 1039);
verifyTermDocs(dir, new Term("content", "aaa"), 1030);
verifyTermDocs(dir, new Term("content", "bbb"), 9);
writer.close();
dir.close();
aux.close();
}
public void testWithPendingDeletes2() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
setUpDirs(dir, aux);
IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
// Adds 10 docs, then replaces them with another 10
// docs, so 10 pending deletes:
for (int i = 0; i < 20; i++) {
Document doc = new Document();
doc.add(newField("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(newField("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
writer.updateDocument(new Term("id", "" + (i%10)), doc);
}
writer.addIndexes(new Directory[] {aux});
// Deletes one of the 10 added docs, leaving 9:
PhraseQuery q = new PhraseQuery();
q.add(new Term("content", "bbb"));
q.add(new Term("content", "14"));
writer.deleteDocuments(q);
writer.forceMerge(1);
writer.commit();
verifyNumDocs(dir, 1039);
verifyTermDocs(dir, new Term("content", "aaa"), 1030);
verifyTermDocs(dir, new Term("content", "bbb"), 9);
writer.close();
dir.close();
aux.close();
}
public void testWithPendingDeletes3() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
setUpDirs(dir, aux);
IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
// Adds 10 docs, then replaces them with another 10
// docs, so 10 pending deletes:
for (int i = 0; i < 20; i++) {
Document doc = new Document();
doc.add(newField("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(newField("content", "bbb " + i, Field.Store.NO,
Field.Index.ANALYZED));
writer.updateDocument(new Term("id", "" + (i%10)), doc);
}
// Deletes one of the 10 added docs, leaving 9:
PhraseQuery q = new PhraseQuery();
q.add(new Term("content", "bbb"));
q.add(new Term("content", "14"));
writer.deleteDocuments(q);
writer.addIndexes(new Directory[] {aux});
writer.forceMerge(1);
writer.commit();
verifyNumDocs(dir, 1039);
verifyTermDocs(dir, new Term("content", "aaa"), 1030);
verifyTermDocs(dir, new Term("content", "bbb"), 9);
writer.close();
dir.close();
aux.close();
}
// case 0: add self or exceed maxMergeDocs, expect exception
public void testAddSelf() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
IndexWriter writer = null;
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
// add 100 documents
addDocs(writer, 100);
assertEquals(100, writer.maxDoc());
writer.close();
writer = newWriter(
aux,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.CREATE).
setMaxBufferedDocs(1000).
setMergePolicy(newLogMergePolicy(false))
);
// add 140 documents in separate files
addDocs(writer, 40);
writer.close();
writer = newWriter(
aux,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.CREATE).
setMaxBufferedDocs(1000).
setMergePolicy(newLogMergePolicy(false))
);
addDocs(writer, 100);
writer.close();
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
try {
// cannot add self
writer.addIndexes(new Directory[] { aux, dir });
assertTrue(false);
}
catch (IllegalArgumentException e) {
assertEquals(100, writer.maxDoc());
}
writer.close();
// make sure the index is correct
verifyNumDocs(dir, 100);
dir.close();
aux.close();
}
// in all the remaining tests, make the doc count of the oldest segment
// in dir large so that it is never merged in addIndexes()
// case 1: no tail segments
public void testNoTailSegments() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
setUpDirs(dir, aux);
IndexWriter writer = newWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(10).
setMergePolicy(newLogMergePolicy(4))
);
addDocs(writer, 10);
writer.addIndexes(new Directory[] { aux });
assertEquals(1040, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
// make sure the index is correct
verifyNumDocs(dir, 1040);
dir.close();
aux.close();
}
// case 2: tail segments, invariants hold, no copy
public void testNoCopySegments() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
setUpDirs(dir, aux);
IndexWriter writer = newWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(9).
setMergePolicy(newLogMergePolicy(4))
);
addDocs(writer, 2);
writer.addIndexes(new Directory[] { aux });
assertEquals(1032, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
// make sure the index is correct
verifyNumDocs(dir, 1032);
dir.close();
aux.close();
}
// case 3: tail segments, invariants hold, copy, invariants hold
public void testNoMergeAfterCopy() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
setUpDirs(dir, aux);
IndexWriter writer = newWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(10).
setMergePolicy(newLogMergePolicy(4))
);
writer.addIndexes(new Directory[] { aux, new MockDirectoryWrapper(random, new RAMDirectory(aux)) });
assertEquals(1060, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
// make sure the index is correct
verifyNumDocs(dir, 1060);
dir.close();
aux.close();
}
// case 4: tail segments, invariants hold, copy, invariants not hold
public void testMergeAfterCopy() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
setUpDirs(dir, aux);
IndexReader reader = IndexReader.open(aux, false);
for (int i = 0; i < 20; i++) {
reader.deleteDocument(i);
}
assertEquals(10, reader.numDocs());
reader.close();
IndexWriter writer = newWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(4).
setMergePolicy(newLogMergePolicy(4))
);
writer.addIndexes(new Directory[] { aux, new MockDirectoryWrapper(random, new RAMDirectory(aux)) });
assertEquals(1020, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
dir.close();
aux.close();
}
// case 5: tail segments, invariants not hold
public void testMoreMerges() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
Directory aux2 = newDirectory();
setUpDirs(dir, aux);
IndexWriter writer = newWriter(
aux2,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.CREATE).
setMaxBufferedDocs(100).
setMergePolicy(newLogMergePolicy(10))
);
writer.setInfoStream(VERBOSE ? System.out : null);
writer.addIndexes(aux);
assertEquals(30, writer.maxDoc());
writer.close();
IndexReader reader = IndexReader.open(aux, false);
for (int i = 0; i < 27; i++) {
reader.deleteDocument(i);
}
assertEquals(3, reader.numDocs());
reader.close();
reader = IndexReader.open(aux2, false);
for (int i = 0; i < 8; i++) {
reader.deleteDocument(i);
}
assertEquals(22, reader.numDocs());
reader.close();
writer = newWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(6).
setMergePolicy(newLogMergePolicy(4))
);
writer.addIndexes(new Directory[] { aux, aux2 });
assertEquals(1040, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
dir.close();
aux.close();
aux2.close();
}
private IndexWriter newWriter(Directory dir, IndexWriterConfig conf)
throws IOException {
conf.setMergePolicy(new LogDocMergePolicy());
final IndexWriter writer = new IndexWriter(dir, conf);
return writer;
}
private void addDocs(IndexWriter writer, int numDocs) throws IOException {
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
doc.add(newField("content", "aaa", Field.Store.NO,
Field.Index.ANALYZED));
writer.addDocument(doc);
}
}
private void addDocs2(IndexWriter writer, int numDocs) throws IOException {
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
doc.add(newField("content", "bbb", Field.Store.NO,
Field.Index.ANALYZED));
writer.addDocument(doc);
}
}
private void verifyNumDocs(Directory dir, int numDocs) throws IOException {
IndexReader reader = IndexReader.open(dir, true);
assertEquals(numDocs, reader.maxDoc());
assertEquals(numDocs, reader.numDocs());
reader.close();
}
private void verifyTermDocs(Directory dir, Term term, int numDocs)
throws IOException {
IndexReader reader = IndexReader.open(dir, true);
TermDocs termDocs = reader.termDocs(term);
int count = 0;
while (termDocs.next())
count++;
assertEquals(numDocs, count);
reader.close();
}
private void setUpDirs(Directory dir, Directory aux) throws IOException {
IndexWriter writer = null;
writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000));
// add 1000 documents in 1 segment
addDocs(writer, 1000);
assertEquals(1000, writer.maxDoc());
assertEquals(1, writer.getSegmentCount());
writer.close();
writer = newWriter(
aux,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.CREATE).
setMaxBufferedDocs(1000).
setMergePolicy(newLogMergePolicy(false, 10))
);
// add 30 documents in 3 segments
for (int i = 0; i < 3; i++) {
addDocs(writer, 10);
writer.close();
writer = newWriter(
aux,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(1000).
setMergePolicy(newLogMergePolicy(false, 10))
);
}
assertEquals(30, writer.maxDoc());
assertEquals(3, writer.getSegmentCount());
writer.close();
}
// LUCENE-1270
public void testHangOnClose() throws IOException {
Directory dir = newDirectory();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
lmp.setUseCompoundFile(false);
lmp.setMergeFactor(100);
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setMaxBufferedDocs(5).setMergePolicy(lmp));
Document doc = new Document();
doc.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for(int i=0;i<60;i++)
writer.addDocument(doc);
Document doc2 = new Document();
doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
for(int i=0;i<10;i++)
writer.addDocument(doc2);
writer.close();
Directory dir2 = newDirectory();
lmp = new LogByteSizeMergePolicy();
lmp.setMinMergeMB(0.0001);
lmp.setUseCompoundFile(false);
lmp.setMergeFactor(4);
writer = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random))
.setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(lmp));
writer.addIndexes(new Directory[] {dir});
writer.close();
dir.close();
dir2.close();
}
// TODO: these are also in TestIndexWriter... add a simple doc-writing method
// like this to LuceneTestCase?
private void addDoc(IndexWriter writer) throws IOException
{
Document doc = new Document();
doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
writer.addDocument(doc);
}
private abstract class RunAddIndexesThreads {
Directory dir, dir2;
final static int NUM_INIT_DOCS = 17;
IndexWriter writer2;
final List<Throwable> failures = new ArrayList<Throwable>();
volatile boolean didClose;
final IndexReader[] readers;
final int NUM_COPY;
final static int NUM_THREADS = 5;
final Thread[] threads = new Thread[NUM_THREADS];
public RunAddIndexesThreads(int numCopy) throws Throwable {
NUM_COPY = numCopy;
dir = new MockDirectoryWrapper(random, new RAMDirectory());
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setMaxBufferedDocs(2));
for (int i = 0; i < NUM_INIT_DOCS; i++)
addDoc(writer);
writer.close();
dir2 = newDirectory();
writer2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
writer2.setInfoStream(VERBOSE ? System.out : null);
writer2.commit();
readers = new IndexReader[NUM_COPY];
for(int i=0;i<NUM_COPY;i++)
readers[i] = IndexReader.open(dir, true);
}
void launchThreads(final int numIter) {
for(int i=0;i<NUM_THREADS;i++) {
threads[i] = new Thread() {
@Override
public void run() {
try {
final Directory[] dirs = new Directory[NUM_COPY];
for(int k=0;k<NUM_COPY;k++)
dirs[k] = new MockDirectoryWrapper(random, new RAMDirectory(dir));
int j=0;
while(true) {
// System.out.println(Thread.currentThread().getName() + ": iter j=" + j);
if (numIter > 0 && j == numIter)
break;
doBody(j++, dirs);
}
} catch (Throwable t) {
handle(t);
}
}
};
}
for(int i=0;i<NUM_THREADS;i++)
threads[i].start();
}
void joinThreads() throws Exception {
for(int i=0;i<NUM_THREADS;i++)
threads[i].join();
}
void close(boolean doWait) throws Throwable {
didClose = true;
writer2.close(doWait);
}
void closeDir() throws Throwable {
for(int i=0;i<NUM_COPY;i++)
readers[i].close();
dir2.close();
}
abstract void doBody(int j, Directory[] dirs) throws Throwable;
abstract void handle(Throwable t);
}
private class CommitAndAddIndexes extends RunAddIndexesThreads {
public CommitAndAddIndexes(int numCopy) throws Throwable {
super(numCopy);
}
@Override
void handle(Throwable t) {
t.printStackTrace(System.out);
synchronized(failures) {
failures.add(t);
}
}
@Override
void doBody(int j, Directory[] dirs) throws Throwable {
switch(j%5) {
case 0:
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(Dir[]) then full merge");
}
writer2.addIndexes(dirs);
writer2.forceMerge(1);
break;
case 1:
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(Dir[])");
}
writer2.addIndexes(dirs);
break;
case 2:
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(IndexReader[])");
}
writer2.addIndexes(readers);
break;
case 3:
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(Dir[]) then maybeMerge");
}
writer2.addIndexes(dirs);
writer2.maybeMerge();
break;
case 4:
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": TEST: commit");
}
writer2.commit();
}
}
}
// LUCENE-1335: test simultaneous addIndexes & commits
// from multiple threads
public void testAddIndexesWithThreads() throws Throwable {
final int NUM_ITER = TEST_NIGHTLY ? 15 : 5;
final int NUM_COPY = 3;
CommitAndAddIndexes c = new CommitAndAddIndexes(NUM_COPY);
c.writer2.setInfoStream(VERBOSE ? System.out : null);
c.launchThreads(NUM_ITER);
for(int i=0;i<100;i++)
addDoc(c.writer2);
c.joinThreads();
int expectedNumDocs = 100+NUM_COPY*(4*NUM_ITER/5)*RunAddIndexesThreads.NUM_THREADS*RunAddIndexesThreads.NUM_INIT_DOCS;
assertEquals(expectedNumDocs, c.writer2.numDocs());
c.close(true);
assertTrue(c.failures.size() == 0);
IndexReader reader = IndexReader.open(c.dir2, true);
assertEquals(expectedNumDocs, reader.numDocs());
reader.close();
c.closeDir();
}
private class CommitAndAddIndexes2 extends CommitAndAddIndexes {
public CommitAndAddIndexes2(int numCopy) throws Throwable {
super(numCopy);
}
@Override
void handle(Throwable t) {
if (!(t instanceof AlreadyClosedException) && !(t instanceof NullPointerException)) {
t.printStackTrace(System.out);
synchronized(failures) {
failures.add(t);
}
}
}
}
// LUCENE-1335: test simultaneous addIndexes & close
public void testAddIndexesWithClose() throws Throwable {
final int NUM_COPY = 3;
CommitAndAddIndexes2 c = new CommitAndAddIndexes2(NUM_COPY);
//c.writer2.setInfoStream(System.out);
c.launchThreads(-1);
// Close w/o first stopping/joining the threads
c.close(true);
//c.writer2.close();
c.joinThreads();
c.closeDir();
assertTrue(c.failures.size() == 0);
}
private class CommitAndAddIndexes3 extends RunAddIndexesThreads {
public CommitAndAddIndexes3(int numCopy) throws Throwable {
super(numCopy);
}
@Override
void doBody(int j, Directory[] dirs) throws Throwable {
switch(j%5) {
case 0:
if (VERBOSE) {
System.out.println("TEST: " + Thread.currentThread().getName() + ": addIndexes + full merge");
}
writer2.addIndexes(dirs);
writer2.forceMerge(1);
break;
case 1:
if (VERBOSE) {
System.out.println("TEST: " + Thread.currentThread().getName() + ": addIndexes");
}
writer2.addIndexes(dirs);
break;
case 2:
if (VERBOSE) {
System.out.println("TEST: " + Thread.currentThread().getName() + ": addIndexes(IR[])");
}
writer2.addIndexes(readers);
break;
case 3:
if (VERBOSE) {
System.out.println("TEST: " + Thread.currentThread().getName() + ": full merge");
}
writer2.forceMerge(1);
break;
case 4:
if (VERBOSE) {
System.out.println("TEST: " + Thread.currentThread().getName() + ": commit");
}
writer2.commit();
}
}
@Override
void handle(Throwable t) {
boolean report = true;
if (t instanceof AlreadyClosedException || t instanceof MergePolicy.MergeAbortedException || t instanceof NullPointerException) {
report = !didClose;
} else if (t instanceof FileNotFoundException) {
report = !didClose;
} else if (t instanceof IOException) {
Throwable t2 = t.getCause();
if (t2 instanceof MergePolicy.MergeAbortedException) {
report = !didClose;
}
}
if (report) {
t.printStackTrace(System.out);
synchronized(failures) {
failures.add(t);
}
}
}
}
// LUCENE-1335: test simultaneous addIndexes & close
public void testAddIndexesWithCloseNoWait() throws Throwable {
final int NUM_COPY = 50;
CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
if (VERBOSE) {
c.writer2.setInfoStream(System.out);
}
c.launchThreads(-1);
Thread.sleep(_TestUtil.nextInt(random, 10, 500));
// Close w/o first stopping/joining the threads
if (VERBOSE) {
System.out.println("TEST: now close(false)");
}
c.close(false);
c.joinThreads();
if (VERBOSE) {
System.out.println("TEST: done join threads");
}
c.closeDir();
assertTrue(c.failures.size() == 0);
}
// LUCENE-1335: test simultaneous addIndexes & close
public void testAddIndexesWithRollback() throws Throwable {
final int NUM_COPY = TEST_NIGHTLY ? 50 : 5;
CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
c.launchThreads(-1);
Thread.sleep(_TestUtil.nextInt(random, 10, 500));
// Close w/o first stopping/joining the threads
if (VERBOSE) {
System.out.println("TEST: now force rollback");
}
c.didClose = true;
c.writer2.rollback();
c.joinThreads();
c.closeDir();
assertTrue(c.failures.size() == 0);
}
// LUCENE-2790: tests that the non CFS files were deleted by addIndexes
public void testNonCFSLeftovers() throws Exception {
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.length; i++) {
dirs[i] = new RAMDirectory();
IndexWriter w = new IndexWriter(dirs[i], new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
Document d = new Document();
d.add(new Field("c", "v", Store.YES, Index.ANALYZED, TermVector.YES));
w.addDocument(d);
w.close();
}
IndexReader[] readers = new IndexReader[] { IndexReader.open(dirs[0]), IndexReader.open(dirs[1]) };
Directory dir = new RAMDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy());
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setNoCFSRatio(1.0); // Force creation of CFS
lmp.setUseCompoundFile(true);
IndexWriter w3 = new IndexWriter(dir, conf);
w3.addIndexes(readers);
w3.close();
assertEquals("Only one compound segment should exist", 3, dir.listAll().length);
}
// LUCENE-2996: tests that addIndexes(IndexReader) applies existing deletes correctly.
public void testExistingDeletes() throws Exception {
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.length; i++) {
dirs[i] = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
IndexWriter writer = new IndexWriter(dirs[i], conf);
Document doc = new Document();
doc.add(new Field("id", "myid", Store.NO, Index.NOT_ANALYZED_NO_NORMS));
writer.addDocument(doc);
writer.close();
}
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
IndexWriter writer = new IndexWriter(dirs[0], conf);
// Now delete the document
writer.deleteDocuments(new Term("id", "myid"));
IndexReader r = IndexReader.open(dirs[1]);
try {
writer.addIndexes(r);
} finally {
r.close();
}
writer.commit();
assertEquals("Documents from the incoming index should not have been deleted", 1, writer.numDocs());
writer.close();
for (Directory dir : dirs) {
dir.close();
}
}
// LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to
// a CFS, given MP preferences
public void testCopyIntoCFS() throws Exception {
// create an index, no CFS (so we can assert that existing segments are not affected)
Directory target = newDirectory();
LogMergePolicy lmp = newLogMergePolicy(false);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null).setMergePolicy(lmp);
IndexWriter w = new IndexWriter(target, conf);
w.addDocument(new Document());
w.commit();
assertFalse(w.segmentInfos.info(0).getUseCompoundFile());
// prepare second index, no-CFS too + .del file + separate norms file
Directory src = newDirectory();
LogMergePolicy lmp2 = newLogMergePolicy(false);
IndexWriterConfig conf2 = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random)).setMergePolicy(lmp2);
IndexWriter w2 = new IndexWriter(src, conf2);
Document doc = new Document();
doc.add(new Field("c", "some text", Store.YES, Index.ANALYZED));
w2.addDocument(doc);
doc = new Document();
doc.add(new Field("d", "delete", Store.NO, Index.NOT_ANALYZED_NO_NORMS));
w2.addDocument(doc);
w2.commit();
w2.deleteDocuments(new Term("d", "delete"));
w2.commit();
w2.close();
// create separate norms file
IndexReader r = IndexReader.open(src, false);
r.setNorm(0, "c", (byte) 1);
r.close();
assertTrue(".del file not found", src.fileExists("_0_1.del"));
assertTrue("separate norms file not found", src.fileExists("_0_1.s0"));
// Case 1: force 'CFS' on target
lmp.setUseCompoundFile(true);
lmp.setNoCFSRatio(1.0);
w.addIndexes(src);
w.commit();
assertFalse("existing segments should not be modified by addIndexes", w.segmentInfos.info(0).getUseCompoundFile());
assertTrue("segment should have been converted to a CFS by addIndexes", w.segmentInfos.info(1).getUseCompoundFile());
assertTrue(".del file not found", target.fileExists("_1_1.del"));
assertTrue("separate norms file not found", target.fileExists("_1_1.s0"));
// Case 2: LMP disallows CFS
lmp.setUseCompoundFile(false);
w.addIndexes(src);
w.commit();
assertFalse("segment should not have been converted to a CFS by addIndexes if MP disallows", w.segmentInfos.info(2).getUseCompoundFile());
w.close();
// cleanup
src.close();
target.close();
}
// LUCENE-3575
public void testFieldNamesChanged() throws IOException {
Directory d1 = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random, d1);
Document doc = new Document();
doc.add(newField("f1", "doc1 field1", Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.add(newField("id", "1", Field.Store.YES, Field.Index.NOT_ANALYZED));
w.addDocument(doc);
IndexReader r1 = w.getReader();
w.close();
Directory d2 = newDirectory();
w = new RandomIndexWriter(random, d2);
doc = new Document();
doc.add(newField("f2", "doc2 field2", Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.add(newField("id", "2", Field.Store.YES, Field.Index.NOT_ANALYZED));
w.addDocument(doc);
IndexReader r2 = w.getReader();
w.close();
Directory d3 = newDirectory();
w = new RandomIndexWriter(random, d3);
w.addIndexes(r1, r2);
r1.close();
d1.close();
r2.close();
d2.close();
IndexReader r3 = w.getReader();
w.close();
assertEquals(2, r3.numDocs());
for(int docID=0;docID<2;docID++) {
Document d = r3.document(docID);
if (d.get("id").equals("1")) {
assertEquals("doc1 field1", d.get("f1"));
} else {
assertEquals("doc2 field2", d.get("f2"));
}
}
r3.close();
d3.close();
}
}