/**********************************************************************************
* $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/test/org/sakaiproject/search/index/soaktest/SearchIndexerNode.java $
* $Id: SearchIndexerNode.java 105078 2012-02-24 23:00:38Z ottenhoff@longsight.com $
***********************************************************************************
*
* Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008, 2009 The Sakai Foundation
*
* Licensed under the Educational Community License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.opensource.org/licenses/ECL-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
**********************************************************************************/
package org.sakaiproject.search.index.soaktest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.sakaiproject.search.api.SearchService;
import org.sakaiproject.search.index.impl.StandardAnalyzerFactory;
import org.sakaiproject.search.indexer.debug.DebugIndexWorkerListener;
import org.sakaiproject.search.indexer.impl.ConcurrentSearchIndexBuilderWorkerImpl;
import org.sakaiproject.search.indexer.impl.JournalManagerUpdateTransaction;
import org.sakaiproject.search.indexer.impl.JournalStorageUpdateTransactionListener;
import org.sakaiproject.search.indexer.impl.SearchBuilderQueueManager;
import org.sakaiproject.search.indexer.impl.TransactionIndexManagerImpl;
import org.sakaiproject.search.indexer.impl.TransactionalIndexWorker;
import org.sakaiproject.search.journal.api.ManagementOperation;
import org.sakaiproject.search.journal.impl.ConcurrentIndexManager;
import org.sakaiproject.search.journal.impl.DbJournalManager;
import org.sakaiproject.search.journal.impl.IndexListenerCloser;
import org.sakaiproject.search.journal.impl.IndexManagementTimerTask;
import org.sakaiproject.search.journal.impl.JournalSettings;
import org.sakaiproject.search.journal.impl.JournaledFSIndexStorage;
import org.sakaiproject.search.journal.impl.JournaledFSIndexStorageUpdateTransactionListener;
import org.sakaiproject.search.journal.impl.MergeUpdateManager;
import org.sakaiproject.search.journal.impl.MergeUpdateOperation;
import org.sakaiproject.search.journal.impl.SharedFilesystemJournalStorage;
import org.sakaiproject.search.mbeans.SearchServiceManagement;
import org.sakaiproject.search.mock.MockClusterService;
import org.sakaiproject.search.mock.MockComponentManager;
import org.sakaiproject.search.mock.MockEventTrackingService;
import org.sakaiproject.search.mock.MockSearchIndexBuilder;
import org.sakaiproject.search.mock.MockSearchService;
import org.sakaiproject.search.mock.MockSecurityService;
import org.sakaiproject.search.mock.MockServerConfigurationService;
import org.sakaiproject.search.mock.MockSessionManager;
import org.sakaiproject.search.mock.MockThreadLocalManager;
import org.sakaiproject.search.mock.MockUserDirectoryService;
import org.sakaiproject.search.optimize.impl.OptimizableIndexImpl;
import org.sakaiproject.search.optimize.impl.OptimizeIndexManager;
import org.sakaiproject.search.optimize.impl.OptimizeIndexOperation;
import org.sakaiproject.search.optimize.impl.OptimizeTransactionListenerImpl;
import org.sakaiproject.search.optimize.shared.impl.DbJournalOptimizationManager;
import org.sakaiproject.search.optimize.shared.impl.JournalOptimizationEndTransactionListener;
import org.sakaiproject.search.optimize.shared.impl.JournalOptimizationManagerImpl;
import org.sakaiproject.search.optimize.shared.impl.JournalOptimizationOperation;
import org.sakaiproject.search.optimize.shared.impl.JournalOptimizationStartTransactionListener;
import org.sakaiproject.search.optimize.shared.impl.OptimizeSharedTransactionListenerImpl;
import org.sakaiproject.search.optimize.shared.impl.SharedFilesystemLoadTransactionListener;
import org.sakaiproject.search.optimize.shared.impl.SharedFilesystemSaveTransactionListener;
import org.sakaiproject.search.transaction.api.TransactionListener;
import org.sakaiproject.search.transaction.impl.LocalTransactionSequenceImpl;
import org.sakaiproject.search.transaction.impl.TransactionSequenceImpl;
import org.sakaiproject.thread_local.api.ThreadLocalManager;
/**
* @author ieb
*/
public class SearchIndexerNode
{
protected static final Log log = LogFactory.getLog(SearchIndexerNode.class);
private SharedTestDataSource tds;
private MergeUpdateOperation mu;
private TransactionalIndexWorker tiw;
private String instanceName;
private String base;
private ConcurrentIndexManager cim;
private String driver;
private String url;
private String userame;
private String password;
private JournaledFSIndexStorage journaledFSIndexStorage;
private MockClusterService clusterService;
private ThreadLocalManager threadLocalManager;
private TransactionSequenceImpl sequence;
private static Random rand = new Random();
public SearchIndexerNode(MockClusterService clusterService, String base,
String instanceName, String driver, String url, String userame,
String password)
{
this.base = base;
this.instanceName = instanceName;
this.driver = driver;
this.url = url;
this.userame = userame;
this.password = password;
this.clusterService = clusterService;
}
public void init() throws Exception
{
String shared = base + "/shared";
String instanceBase = base + "/" + instanceName;
String localIndexBase = instanceBase + "/index/local";
String sharedJournalBase = shared;
threadLocalManager = new MockThreadLocalManager();
JournalSettings journalSettings = new JournalSettings();
journalSettings.setLocalIndexBase(localIndexBase);
journalSettings.setSharedJournalBase(sharedJournalBase);
journalSettings.setMinimumOptimizeSavePoints(50); // the number of
// shared save
// points before we
// attemt to merge
journalSettings.setOptimizeMergeSize(50); // the number of local save
// points before we try to
// merge
journalSettings.setLocalMaxBufferedDocs(50);
journalSettings.setLocalMaxMergeDocs(1000000);
journalSettings.setLocalMaxMergeFactor(10);
journalSettings.setSharedMaxBufferedDocs(50);
journalSettings.setSharedMaxMergeDocs(1000000);
journalSettings.setSharedMaxMergeFactor(10);
journalSettings.setCreateMaxBufferedDocs(50);
journalSettings.setCreateMaxMergeDocs(1000000);
journalSettings.setCreateMaxMergeFactor(10);
journalSettings.setSoakTest(true);
tds = new SharedTestDataSource(base, 10, false, driver, url, userame, password);
mu = new MergeUpdateOperation();
journaledFSIndexStorage = new JournaledFSIndexStorage();
journaledFSIndexStorage.setThreadLocalManager(threadLocalManager);
StandardAnalyzerFactory analyzerFactory = new StandardAnalyzerFactory();
DbJournalManager journalManager = new DbJournalManager();
MockServerConfigurationService serverConfigurationService = new MockServerConfigurationService();
serverConfigurationService.setInstanceName(instanceName);
MergeUpdateManager mergeUpdateManager = new MergeUpdateManager();
sequence = new TransactionSequenceImpl();
SharedFilesystemJournalStorage sharedFilesystemJournalStorage = new SharedFilesystemJournalStorage();
JournaledFSIndexStorageUpdateTransactionListener journaledFSIndexStorageUpdateTransactionListener = new JournaledFSIndexStorageUpdateTransactionListener();
sequence.setDatasource(tds.getDataSource());
sequence.setName("TransactionalIndexWorkerTest");
journaledFSIndexStorageUpdateTransactionListener
.setJournaledIndex(journaledFSIndexStorage);
journaledFSIndexStorageUpdateTransactionListener
.setJournalManager(journalManager);
journaledFSIndexStorageUpdateTransactionListener
.setJournalStorage(sharedFilesystemJournalStorage);
sharedFilesystemJournalStorage.setJournalSettings(journalSettings);
sequence.setDatasource(tds.getDataSource());
mergeUpdateManager
.addTransactionListener(journaledFSIndexStorageUpdateTransactionListener);
mergeUpdateManager.setSequence(sequence);
journalManager.setDatasource(tds.getDataSource());
journalManager.setServerConfigurationService(serverConfigurationService);
journaledFSIndexStorage.setAnalyzerFactory(analyzerFactory);
journaledFSIndexStorage.setDatasource(tds.getDataSource());
journaledFSIndexStorage.setJournalManager(journalManager);
journaledFSIndexStorage.setRecoverCorruptedIndex(false);
journaledFSIndexStorage.setJournalSettings(journalSettings);
journaledFSIndexStorage.setServerConfigurationService(serverConfigurationService);
mu.setJournaledObject(journaledFSIndexStorage);
mu.setMergeUpdateManager(mergeUpdateManager);
OptimizableIndexImpl optimizableIndex = new OptimizableIndexImpl();
optimizableIndex.setJournaledIndex(journaledFSIndexStorage);
OptimizeTransactionListenerImpl otli = new OptimizeTransactionListenerImpl();
otli.setJournalSettings(journalSettings);
otli.setOptimizableIndex(optimizableIndex);
OptimizeIndexManager optimizeUpdateManager = new OptimizeIndexManager();
optimizeUpdateManager.setAnalyzerFactory(analyzerFactory);
optimizeUpdateManager.setJournalSettings(journalSettings);
optimizeUpdateManager.addTransactionListener(otli);
optimizeUpdateManager.setSequence(new LocalTransactionSequenceImpl());
mu.setOptimizeUpdateManager(optimizeUpdateManager);
JournalManagerUpdateTransaction journalManagerUpdateTransaction = new JournalManagerUpdateTransaction();
MockSearchIndexBuilder mockSearchIndexBuilder = new MockSearchIndexBuilder();
TransactionIndexManagerImpl transactionIndexManager = new TransactionIndexManagerImpl();
SearchBuilderQueueManager searchBuilderQueueManager = new SearchBuilderQueueManager();
TransactionSequenceImpl lockSequenceImpl = new TransactionSequenceImpl();
lockSequenceImpl.setDatasource(tds.getDataSource());
lockSequenceImpl.setName("queueManagerLock");
lockSequenceImpl.setMinValue(2000);
lockSequenceImpl.setMaxValue(10000000);
lockSequenceImpl.init();
searchBuilderQueueManager.setSequence(lockSequenceImpl);
transactionIndexManager.setAnalyzerFactory(new StandardAnalyzerFactory());
transactionIndexManager.setJournalSettings(journalSettings);
transactionIndexManager.setSequence(sequence);
journalManagerUpdateTransaction.setJournalManager(journalManager);
JournalStorageUpdateTransactionListener journalStorageUpdateTransactionListener = new JournalStorageUpdateTransactionListener();
journalStorageUpdateTransactionListener
.setJournalStorage(sharedFilesystemJournalStorage);
searchBuilderQueueManager.setDatasource(tds.getDataSource());
searchBuilderQueueManager.setSearchIndexBuilder(mockSearchIndexBuilder);
transactionIndexManager
.addTransactionListener(journalStorageUpdateTransactionListener);
transactionIndexManager.addTransactionListener(searchBuilderQueueManager);
transactionIndexManager.addTransactionListener(journalManagerUpdateTransaction);
// transactionIndexManager.addTransactionListener(new
// DebugTransactionListener());
MockSearchService searchService = new MockSearchService();
searchService.setDatasource(tds.getDataSource());
searchService.setServerConfigurationService(new MockServerConfigurationService());
tiw = new TransactionalIndexWorker();
tiw.setSearchIndexBuilder(mockSearchIndexBuilder);
tiw.setServerConfigurationService(serverConfigurationService);
tiw.setTransactionIndexManager(transactionIndexManager);
tiw.setSearchService(searchService);
tiw.setThreadLocalManager(threadLocalManager);
// tiw.addIndexWorkerDocumentListener(new
// DebugIndexWorkerDocumentListener());
tiw.addIndexWorkerListener(new DebugIndexWorkerListener());
sequence.init();
searchBuilderQueueManager.init();
transactionIndexManager.init();
journalManager.init();
mu.init();
journaledFSIndexStorage.init();
tiw.init();
OptimizeIndexOperation oo = new OptimizeIndexOperation();
oo.setJournaledObject(journaledFSIndexStorage);
oo.setOptimizeUpdateManager(optimizeUpdateManager);
oo.init();
cim = new ConcurrentIndexManager();
IndexManagementTimerTask indexer = new IndexManagementTimerTask();
IndexManagementTimerTask merger = new IndexManagementTimerTask();
IndexManagementTimerTask optimizer = new IndexManagementTimerTask();
indexer.setThreadLocalManager(threadLocalManager);
merger.setThreadLocalManager(threadLocalManager);
optimizer.setThreadLocalManager(threadLocalManager);
MockComponentManager componentManager = new MockComponentManager();
MockEventTrackingService eventTrackingService = new MockEventTrackingService();
MockSessionManager sessionManager = new MockSessionManager();
MockUserDirectoryService userDirectoryService = new MockUserDirectoryService();
MockSecurityService securityService = new MockSecurityService();
ConcurrentSearchIndexBuilderWorkerImpl csibw = new ConcurrentSearchIndexBuilderWorkerImpl();
csibw.setComponentManager(componentManager);
csibw.setEventTrackingService(eventTrackingService);
csibw.setIndexWorker(tiw);
csibw.setSearchService(searchService);
csibw.setSecurityService(securityService);
csibw.setJournalSettings(journalSettings);
csibw.setUserDirectoryService(userDirectoryService);
csibw.setServerConfigurationService(serverConfigurationService);
csibw.init();
indexer.setManagementOperation(csibw);
merger.setManagementOperation(mu);
optimizer.setManagementOperation(oo);
List<IndexManagementTimerTask> taskList = new ArrayList<IndexManagementTimerTask>();
taskList.add(indexer);
indexer.setDelay(10);
indexer.setPeriod(1000);
taskList.add(merger);
merger.setDelay(10);
merger.setPeriod(10000);
taskList.add(optimizer);
optimizer.setDelay(50);
optimizer.setPeriod(10000);
IndexManagementTimerTask docloader = new IndexManagementTimerTask();
docloader.setThreadLocalManager(threadLocalManager);
docloader.setDelay(5);
docloader.setPeriod(500);
docloader.setManagementOperation(new ManagementOperation()
{
public void runOnce()
{
try
{
tds.populateDocuments(500, instanceName);
tds.resetAfter(3000);
}
catch (Exception e)
{
log.error("Failed to LoadDocuments ", e);
}
}
});
taskList.add(docloader);
journaledFSIndexStorage.addIndexListener(cim);
IndexListenerCloser indexCloser = new IndexListenerCloser();
journaledFSIndexStorage.addIndexListener(indexCloser);
// Journal Optimization
JournalOptimizationOperation journalOptimizationOperation = new JournalOptimizationOperation();
JournalOptimizationManagerImpl journalOptimizationManager = new JournalOptimizationManagerImpl();
JournalOptimizationStartTransactionListener journalOptimizationStartTransactionListener = new JournalOptimizationStartTransactionListener();
JournalOptimizationEndTransactionListener journalOptimizationEndTransactionListener = new JournalOptimizationEndTransactionListener();
SharedFilesystemLoadTransactionListener sharedFilesystemLoadTransactionListener = new SharedFilesystemLoadTransactionListener();
SharedFilesystemSaveTransactionListener sharedFilesystemSaveTransactionListener = new SharedFilesystemSaveTransactionListener();
sharedFilesystemSaveTransactionListener.setSharedSleep(120000);
OptimizeSharedTransactionListenerImpl optimizeSharedTransactionListener = new OptimizeSharedTransactionListenerImpl();
optimizeSharedTransactionListener.setJournalSettings(journalSettings);
DbJournalOptimizationManager optimizeJournalManager = new DbJournalOptimizationManager();
TransactionSequenceImpl optimizeSequence = new TransactionSequenceImpl();
sequence.getClass();
clusterService.addServerConfigurationService(serverConfigurationService);
long journalOptimizeLimit = 100;
optimizeSequence.setName("journaloptimize");
optimizeSequence.setDatasource(tds.getDataSource());
optimizeJournalManager.setClusterService(clusterService);
optimizeJournalManager.setDatasource(tds.getDataSource());
optimizeJournalManager.setJournalSettings(journalSettings);
optimizeJournalManager.setServerConfigurationService(serverConfigurationService);
sharedFilesystemLoadTransactionListener
.setSharedFilesystemJournalStorage(sharedFilesystemJournalStorage);
sharedFilesystemSaveTransactionListener
.setSharedFilesystemJournalStorage(sharedFilesystemJournalStorage);
journalOptimizationManager
.addTransactionListener(journalOptimizationStartTransactionListener);
journalOptimizationManager
.addTransactionListener(sharedFilesystemLoadTransactionListener);
journalOptimizationManager
.addTransactionListener(optimizeSharedTransactionListener);
journalOptimizationManager
.addTransactionListener(sharedFilesystemSaveTransactionListener);
journalOptimizationManager
.addTransactionListener(journalOptimizationEndTransactionListener);
journalOptimizationManager.setAnalyzerFactory(analyzerFactory);
journalOptimizationManager.setJournalManager(optimizeJournalManager);
journalOptimizationManager.setSequence(optimizeSequence);
journalOptimizationManager.setJournalSettings(journalSettings);
List<TransactionListener> tl = new ArrayList<TransactionListener>();
tl.add(journalOptimizationStartTransactionListener);
tl.add(sharedFilesystemLoadTransactionListener);
tl.add(optimizeSharedTransactionListener);
tl.add(sharedFilesystemSaveTransactionListener);
tl.add(journalOptimizationEndTransactionListener);
journalOptimizationManager.setTransactionListeners(tl);
journalOptimizationOperation
.setJournalOptimizationManager(journalOptimizationManager);
journalOptimizationOperation.setServerConfigurationService(serverConfigurationService);
optimizeJournalManager.init();
optimizeSequence.init();
journalOptimizationOperation.init();
IndexManagementTimerTask journalOptimizer = new IndexManagementTimerTask();
journalOptimizer.setThreadLocalManager(threadLocalManager);
journalOptimizer.setManagementOperation(journalOptimizationOperation);
journalOptimizer.setDelay(10);
journalOptimizer.setPeriod(1000);
taskList.add(journalOptimizer);
SearchServiceManagement mbean = new SearchServiceManagement(instanceName);
mbean.setIndexStorageProvider(journaledFSIndexStorage);
mbean.setSearchService(searchService);
mbean.setIndexListenerCloser(indexCloser);
mbean.setIndexWorker(tiw);
mbean.setThreadLocalManager(threadLocalManager);
mbean.init();
cim.setSearchService(searchService);
cim.setTasks(taskList);
cim.init();
}
/**
* @throws Exception
*/
public void close() throws Exception
{
cim.close();
}
public void cleanup() throws Exception
{
cim.cleanup();
tds.close();
}
/**
* @throws IOException
*/
public void testSearch()
{
try
{
long start1 = System.currentTimeMillis();
IndexSearcher is = journaledFSIndexStorage.getIndexSearcher();
TermQuery tq = new TermQuery(new Term(SearchService.FIELD_CONTENTS, "node"));
long start = System.currentTimeMillis();
TopDocs h = is.search(tq, 1000);
long end = System.currentTimeMillis();
log.debug("Got " + h.totalHits + " hits from " + is.getIndexReader().numDocs()
+ " for node " + instanceName + " in " + (end - start) + ":"
+ (start - start1) + " ms");
}
catch (Exception ex)
{
log.error("Search Failed with, perhapse due to a file being removed "
+ ex.getMessage());
}
}
public void testSlowSearch() throws Exception
{
long start1 = System.currentTimeMillis();
log.debug("Getting index searcher");
IndexSearcher is = journaledFSIndexStorage.getIndexSearcher();
TermQuery tq = new TermQuery(new Term(SearchService.FIELD_CONTENTS, "node"));
long start = System.currentTimeMillis();
log.info("Searching with " + is + " and reader " + is.getIndexReader());
TopDocs topdocs = is.search(tq, 100000);
log.info("Performing Search and Sleeping 500ms with " + is);
Thread.sleep(500);
log.info("Performing Search and Sleeping 500ms with " + is);
long end = System.currentTimeMillis();
log.info("Got " + topdocs.totalHits + " hits from " + is.getIndexReader().numDocs()
+ " for node " + instanceName + " in " + (end - start) + ":"
+ (start - start1) + " ms");
for (int i = 0; i < topdocs.totalHits; i++)
{
Document d = is.doc(i);
List<Field> e = d.getFields();
for (int q = 0; q < e.size(); q++)
{
e.get(q);
}
}
}
/**
* @throws IOException
*/
public void reopenIndex() throws IOException
{
journaledFSIndexStorage.markModified();
journaledFSIndexStorage.getIndexReader();
log.info("Reopened Index");
}
/**
* @return
*/
public ThreadLocalManager getThreadLocalManager()
{
return threadLocalManager;
}
/**
*
*/
public void makeTransactionHole()
{
if ((rand.nextInt(100) % 18) == 0)
{
long n = sequence.getNextId();
log.info("Simulated Transaction Hole at " + n);
}
}
}