package org.apache.lucene.index; /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.util.LuceneTestCase; import java.io.IOException; public class TestConcurrentMergeScheduler extends LuceneTestCase { private class FailOnlyOnFlush extends MockDirectoryWrapper.Failure { boolean doFail; boolean hitExc; @Override public void setDoFail() { this.doFail = true; hitExc = false; } @Override public void clearDoFail() { this.doFail = false; } @Override public void eval(MockDirectoryWrapper dir) throws IOException { if (doFail && isTestThread()) { boolean isDoFlush = false; boolean isClose = false; StackTraceElement[] trace = new Exception().getStackTrace(); for (int i = 0; i < trace.length; i++) { if ("doFlush".equals(trace[i].getMethodName())) { isDoFlush = true; } if ("close".equals(trace[i].getMethodName())) { isClose = true; } } if (isDoFlush && !isClose && random.nextBoolean()) { hitExc = true; throw new IOException(Thread.currentThread().getName() + ": now failing during flush"); } } } } // Make sure running BG merges still work fine even when // we are hitting exceptions during flushing. public void testFlushExceptions() throws IOException { MockDirectoryWrapper directory = newDirectory(); FailOnlyOnFlush failure = new FailOnlyOnFlush(); directory.failOn(failure); IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2)); writer.setInfoStream(VERBOSE ? System.out : null); Document doc = new Document(); Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.add(idField); int extraCount = 0; for(int i=0;i<10;i++) { if (VERBOSE) { System.out.println("TEST: iter=" + i); } for(int j=0;j<20;j++) { idField.setValue(Integer.toString(i*20+j)); writer.addDocument(doc); } // must cycle here because sometimes the merge flushes // the doc we just added and so there's nothing to // flush, and we don't hit the exception while(true) { writer.addDocument(doc); failure.setDoFail(); try { writer.flush(true, true); if (failure.hitExc) { fail("failed to hit IOException"); } extraCount++; } catch (IOException ioe) { if (VERBOSE) { ioe.printStackTrace(System.out); } failure.clearDoFail(); break; } } assertEquals(20*(i+1)+extraCount, writer.numDocs()); } writer.close(); IndexReader reader = IndexReader.open(directory, true); assertEquals(200+extraCount, reader.numDocs()); reader.close(); directory.close(); } // Test that deletes committed after a merge started and // before it finishes, are correctly merged back: public void testDeleteMerging() throws IOException { MockDirectoryWrapper directory = newDirectory(); LogDocMergePolicy mp = new LogDocMergePolicy(); // Force degenerate merging so we can get a mix of // merging of segments with and without deletes at the // start: mp.setMinMergeDocs(1000); IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)) .setMergePolicy(mp)); writer.setInfoStream(VERBOSE ? System.out : null); Document doc = new Document(); Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.add(idField); for(int i=0;i<10;i++) { if (VERBOSE) { System.out.println("\nTEST: cycle"); } for(int j=0;j<100;j++) { idField.setValue(Integer.toString(i*100+j)); writer.addDocument(doc); } int delID = i; while(delID < 100*(1+i)) { if (VERBOSE) { System.out.println("TEST: del " + delID); } writer.deleteDocuments(new Term("id", ""+delID)); delID += 10; } writer.commit(); } writer.close(); IndexReader reader = IndexReader.open(directory, true); // Verify that we did not lose any deletes... assertEquals(450, reader.numDocs()); reader.close(); directory.close(); } public void testNoExtraFiles() throws IOException { MockDirectoryWrapper directory = newDirectory(); IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)) .setMaxBufferedDocs(2)); writer.setInfoStream(VERBOSE ? System.out : null); for(int iter=0;iter<7;iter++) { if (VERBOSE) { System.out.println("TEST: iter=" + iter); } for(int j=0;j<21;j++) { Document doc = new Document(); doc.add(newField("content", "a b c", Field.Store.NO, Field.Index.ANALYZED)); writer.addDocument(doc); } writer.close(); TestIndexWriter.assertNoUnreferencedFiles(directory, "testNoExtraFiles"); // Reopen writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)) .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(2)); writer.setInfoStream(VERBOSE ? System.out : null); } writer.close(); directory.close(); } public void testNoWaitClose() throws IOException { MockDirectoryWrapper directory = newDirectory(); Document doc = new Document(); Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED); doc.add(idField); IndexWriter writer = new IndexWriter( directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setMaxBufferedDocs(2). setMergePolicy(newLogMergePolicy(100)) ); for(int iter=0;iter<10;iter++) { for(int j=0;j<201;j++) { idField.setValue(Integer.toString(iter*201+j)); writer.addDocument(doc); } int delID = iter*201; for(int j=0;j<20;j++) { writer.deleteDocuments(new Term("id", Integer.toString(delID))); delID += 5; } // Force a bunch of merge threads to kick off so we // stress out aborting them on close: ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(3); writer.addDocument(doc); writer.commit(); writer.close(false); IndexReader reader = IndexReader.open(directory, true); assertEquals((1+iter)*182, reader.numDocs()); reader.close(); // Reopen writer = new IndexWriter( directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.APPEND). setMergePolicy(newLogMergePolicy(100)) ); } writer.close(); directory.close(); } }