/* $Id: ExpireStufferThread.java 988245 2010-08-23 18:39:35Z kwright $ */
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.crawler.system;
import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.*;
import org.apache.manifoldcf.crawler.system.Logging;
import java.util.*;
import java.lang.reflect.*;
/** This class represents the stuffer thread. This thread's job is to request documents from the database and add them to the
* document queue. The thread then sleeps until the document queue is empty again.
*/
public class ExpireStufferThread extends Thread
{
public static final String _rcsid = "@(#)$Id: ExpireStufferThread.java 988245 2010-08-23 18:39:35Z kwright $";
// Local data
/** This is a reference to the static main document expiration queue */
protected final DocumentCleanupQueue documentQueue;
/** Worker thread pool reset manager */
protected final WorkerResetManager resetManager;
/** This is the number of entries we want to stuff at any one time. */
protected final int n;
/** Process ID */
protected final String processID;
/** Constructor.
*@param documentQueue is the document queue we'll be stuffing.
*@param n represents the number of threads that will be processing queued stuff, NOT the
* number of documents to be done at once!
*/
public ExpireStufferThread(DocumentCleanupQueue documentQueue, int n, WorkerResetManager resetManager, String processID)
throws ManifoldCFException
{
super();
this.documentQueue = documentQueue;
this.n = n;
this.resetManager = resetManager;
this.processID = processID;
setName("Expire stuffer thread");
setDaemon(true);
// The priority of this thread is higher than most others. We want stuffing to proceed even if the machine
// is pretty busy already.
setPriority(getPriority()+1);
}
public void run()
{
resetManager.registerMe();
try
{
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
IJobManager jobManager = JobManagerFactory.make(threadContext);
Logging.threads.debug("Expire stuffer thread: Maximum document count per check is "+Integer.toString(n));
// Hashmap keyed by jobid and containing ArrayLists.
// This way we can guarantee priority will do the right thing, because the
// priority is per-job. We CANNOT guarantee anything about scheduling order, however,
// other than that it falls in the time window.
HashMap documentSets = new HashMap();
// Job description map (local) - designed to improve performance.
// Cleared and reloaded on every batch of documents.
HashMap jobDescriptionMap = new HashMap();
IDBInterface database = DBInterfaceFactory.make(threadContext,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
int deleteChunkSize = database.getMaxInClause();
// Loop
while (true)
{
// Do another try/catch around everything in the loop
try
{
if (Thread.currentThread().isInterrupted())
throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
// Check if we're okay
resetManager.waitForReset(threadContext);
// System.out.println("Waiting...");
// Wait until queue is empty enough.
boolean isEmpty = documentQueue.checkIfEmpty(n*3);
if (isEmpty == false)
{
sleep(1000);
continue;
}
Logging.threads.debug("Expiration stuffer thread woke up");
// What we want to do is load enough documents to completely fill n queued document sets.
// The number n passed in here thus cannot be used in a query to limit the number of returned
// results. Instead, it must be factored into the limit portion of the query.
long currentTime = System.currentTimeMillis();
DocumentSetAndFlags docsAndFlags = jobManager.getExpiredDocuments(processID,deleteChunkSize,currentTime);
DocumentDescription[] descs = docsAndFlags.getDocumentSet();
boolean[] deleteFromIndex = docsAndFlags.getFlags();
if (Thread.currentThread().isInterrupted())
throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
if (Logging.threads.isDebugEnabled())
{
Logging.threads.debug("Expiration stuffer thread: Found "+Integer.toString(descs.length)+" documents to expire");
}
// If there are no documents at all, then we can sleep for a while.
// The theory is that we need to allow stuff to accumulate.
if (descs.length == 0)
{
ManifoldCF.sleep(5000L); // 5 seconds
continue;
}
// Do the stuffing. Each set must be segregated by job, since we need the job ID in the doc set.
Map jobMap = new HashMap();
int k = 0;
while (k < descs.length)
{
CleanupQueuedDocument x = new CleanupQueuedDocument(descs[k],deleteFromIndex[k]);
Long jobID = descs[k].getJobID();
List y = (List)jobMap.get(jobID);
if (y == null)
{
y = new ArrayList();
jobMap.put(jobID,y);
}
y.add(x);
k++;
}
Iterator iter = jobMap.keySet().iterator();
while (iter.hasNext())
{
Long jobID = (Long)iter.next();
IJobDescription jobDescription = jobManager.load(jobID,true);
List y = (List)jobMap.get(jobID);
CleanupQueuedDocument[] docDescs = new CleanupQueuedDocument[y.size()];
k = 0;
while (k < docDescs.length)
{
docDescs[k] = (CleanupQueuedDocument)y.get(k);
k++;
}
DocumentCleanupSet set = new DocumentCleanupSet(docDescs,jobDescription);
documentQueue.addDocuments(set);
}
yield();
}
catch (ManifoldCFException e)
{
if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
break;
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
{
resetManager.noteEvent();
Logging.threads.error("Expiration stuffer thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
try
{
// Give the database a chance to catch up/wake up
ManifoldCF.sleep(10000L);
}
catch (InterruptedException se)
{
break;
}
continue;
}
// Log it, but keep the thread alive
Logging.threads.error("Exception tossed: "+e.getMessage(),e);
if (e.getErrorCode() == ManifoldCFException.SETUP_ERROR)
{
System.exit(1);
}
}
catch (InterruptedException e)
{
// We're supposed to quit
break;
}
catch (OutOfMemoryError e)
{
System.err.println("agents process ran out of memory - shutting down");
e.printStackTrace(System.err);
System.exit(-200);
}
catch (Throwable e)
{
// A more severe error - but stay alive
Logging.threads.fatal("Error tossed: "+e.getMessage(),e);
}
}
}
catch (Throwable e)
{
// Severe error on initialization
System.err.println("agents process could not start - shutting down");
Logging.threads.fatal("ExpirationStufferThread initialization error tossed: "+e.getMessage(),e);
System.exit(-300);
}
}
}