package org.apache.tika.batch;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Date;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.SafeContentHandler;
import org.apache.tika.sax.ToXMLContentHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
/**
* This is a base class for file consumers. The
* goal of this class is to abstract out the multithreading
* and recordkeeping components.
* <p/>
*/
public abstract class FileResourceConsumer implements Callable<IFileProcessorFutureResult> {
protected static final Logger LOG = LoggerFactory.getLogger(FileResourceConsumer.class);
private enum STATE {
NOT_YET_STARTED,
ACTIVELY_CONSUMING,
SWALLOWED_POISON,
THREAD_INTERRUPTED,
EXCEEDED_MAX_CONSEC_WAIT_MILLIS,
ASKED_TO_SHUTDOWN,
TIMED_OUT,
CONSUMER_EXCEPTION,
CONSUMER_ERROR,
COMPLETED
}
public static String TIMED_OUT = "timed_out";
public static String OOM = "oom";
public static String IO_IS = "io_on_inputstream";
public static String IO_OS = "io_on_outputstream";
public static String PARSE_ERR = "parse_err";
public static String PARSE_EX = "parse_ex";
public static String ELAPSED_MILLIS = "elapsedMS";
private static AtomicInteger numConsumers = new AtomicInteger(-1);
private long maxConsecWaitInMillis = 10*60*1000;// 10 minutes
private final ArrayBlockingQueue<FileResource> fileQueue;
private final int consumerId;
//used to lock checks on state to prevent
private final Object lock = new Object();
//this records the file that is currently
//being processed. It is null if no file is currently being processed.
//no need for volatile because of lock for checkForStales
private FileStarted currentFile = null;
//total number of files consumed; volatile so that reporter
//sees the latest
private volatile int numResourcesConsumed = 0;
//total number of exceptions that were handled by subclasses;
//volatile so that reporter sees the latest
private volatile int numHandledExceptions = 0;
//after this has been set to ACTIVELY_CONSUMING,
//this should only be set by setEndedState.
private volatile STATE currentState = STATE.NOT_YET_STARTED;
public FileResourceConsumer(ArrayBlockingQueue<FileResource> fileQueue) {
this.fileQueue = fileQueue;
consumerId = numConsumers.incrementAndGet();
}
public IFileProcessorFutureResult call() {
currentState = STATE.ACTIVELY_CONSUMING;
try {
FileResource fileResource = getNextFileResource();
while (fileResource != null) {
LOG.trace("file consumer is about to process: {}", fileResource.getResourceId());
boolean consumed = _processFileResource(fileResource);
LOG.trace("file consumer has finished processing: {}", fileResource.getResourceId());
if (consumed) {
numResourcesConsumed++;
}
fileResource = getNextFileResource();
}
} catch (InterruptedException e) {
setEndedState(STATE.THREAD_INTERRUPTED);
}
setEndedState(STATE.COMPLETED);
return new FileConsumerFutureResult(currentFile, numResourcesConsumed);
}
/**
* Main piece of code that needs to be implemented. Clients
* are responsible for closing streams and handling the exceptions
* that they'd like to handle.
* <p/>
* Unchecked throwables can be thrown past this, of course. When an unchecked
* throwable is thrown, this logs the error, and then rethrows the exception.
* Clients/subclasses should make sure to catch and handle everything they can.
* <p/>
* The design goal is that the whole process should close up and shutdown soon after
* an unchecked exception or error is thrown.
* <p/>
* Make sure to call {@link #incrementHandledExceptions()} appropriately in
* your implementation of this method.
* <p/>
*
* @param fileResource resource to process
* @return whether or not a file was successfully processed
*/
public abstract boolean processFileResource(FileResource fileResource);
/**
* Make sure to call this appropriately!
*/
protected void incrementHandledExceptions() {
numHandledExceptions++;
}
/**
* Returns whether or not the consumer is still could process
* a file or is still processing a file (ACTIVELY_CONSUMING or ASKED_TO_SHUTDOWN)
* @return whether this consumer is still active
*/
public boolean isStillActive() {
if (Thread.currentThread().isInterrupted()) {
return false;
} else if( currentState == STATE.NOT_YET_STARTED ||
currentState == STATE.ACTIVELY_CONSUMING ||
currentState == STATE.ASKED_TO_SHUTDOWN) {
return true;
}
return false;
}
private boolean _processFileResource(FileResource fileResource) {
currentFile = new FileStarted(fileResource.getResourceId());
boolean consumed = false;
try {
consumed = processFileResource(fileResource);
} catch (RuntimeException e) {
setEndedState(STATE.CONSUMER_EXCEPTION);
throw e;
} catch (Error e) {
setEndedState(STATE.CONSUMER_ERROR);
throw e;
}
//if anything is thrown from processFileResource, then the fileStarted
//will remain what it was right before the exception was thrown.
currentFile = null;
return consumed;
}
/**
* This politely asks the consumer to shutdown.
* Before processing another file, the consumer will check to see
* if it has been asked to terminate.
* <p>
* This offers another method for politely requesting
* that a FileResourceConsumer stop processing
* besides passing it {@link org.apache.tika.batch.PoisonFileResource}.
*
*/
public void pleaseShutdown() {
setEndedState(STATE.ASKED_TO_SHUTDOWN);
}
/**
* Returns the name and start time of a file that is currently being processed.
* If no file is currently being processed, this will return null.
*
* @return FileStarted or null
*/
public FileStarted getCurrentFile() {
return currentFile;
}
public int getNumResourcesConsumed() {
return numResourcesConsumed;
}
public int getNumHandledExceptions() {
return numHandledExceptions;
}
/**
* Checks to see if the currentFile being processed (if there is one)
* should be timed out (still being worked on after staleThresholdMillis).
* <p>
* If the consumer should be timed out, this will return the currentFile and
* set the state to TIMED_OUT.
* <p>
* If the consumer was already timed out earlier or
* is not processing a file or has been working on a file
* for less than #staleThresholdMillis, then this will return null.
* <p>
* @param staleThresholdMillis threshold to determine whether the consumer has gone stale.
* @return null or the file started that triggered the stale condition
*/
public FileStarted checkForTimedOutMillis(long staleThresholdMillis) {
//if there isn't a current file, don't bother obtaining lock
if (currentFile == null) {
return null;
}
//if threshold is < 0, don't even look.
if (staleThresholdMillis < 0) {
return null;
}
synchronized(lock) {
//check again once the lock has been obtained
if (currentState != STATE.ACTIVELY_CONSUMING
&& currentState != STATE.ASKED_TO_SHUTDOWN) {
return null;
}
FileStarted tmp = currentFile;
if (tmp == null) {
return null;
}
if (tmp.getElapsedMillis() > staleThresholdMillis) {
setEndedState(STATE.TIMED_OUT);
LOG.error("{}", getXMLifiedLogMsg(
TIMED_OUT,
tmp.getResourceId(),
ELAPSED_MILLIS, Long.toString(tmp.getElapsedMillis())));
return tmp;
}
}
return null;
}
protected String getXMLifiedLogMsg(String type, String resourceId, String... attrs) {
return getXMLifiedLogMsg(type, resourceId, null, attrs);
}
/**
* Use this for structured output that captures resourceId and other attributes.
*
* @param type entity name for exception
* @param resourceId resourceId string
* @param t throwable can be null
* @param attrs (array of key0, value0, key1, value1, etc.)
*/
protected String getXMLifiedLogMsg(String type, String resourceId, Throwable t, String... attrs) {
ContentHandler toXML = new ToXMLContentHandler();
SafeContentHandler handler = new SafeContentHandler(toXML);
AttributesImpl attributes = new AttributesImpl();
attributes.addAttribute("", "resourceId", "resourceId", "", resourceId);
for (int i = 0; i < attrs.length - 1; i++) {
attributes.addAttribute("", attrs[i], attrs[i], "", attrs[i + 1]);
}
try {
handler.startDocument();
handler.startElement("", type, type, attributes);
if (t != null) {
StringWriter stackWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stackWriter);
t.printStackTrace(printWriter);
printWriter.flush();
stackWriter.flush();
char[] chars = stackWriter.toString().toCharArray();
handler.characters(chars, 0, chars.length);
}
handler.endElement("", type, type);
handler.endDocument();
} catch (SAXException e) {
LOG.warn("error writing xml stream for: {}", resourceId, t);
}
return handler.toString();
}
private FileResource getNextFileResource() throws InterruptedException {
FileResource fileResource = null;
long start = new Date().getTime();
while (fileResource == null) {
//check to see if thread is interrupted before polling
if (Thread.currentThread().isInterrupted()) {
setEndedState(STATE.THREAD_INTERRUPTED);
LOG.debug("Consumer thread was interrupted.");
break;
}
synchronized(lock) {
//need to lock here to prevent race condition with other threads setting state
if (currentState != STATE.ACTIVELY_CONSUMING) {
LOG.debug("Consumer already closed because of: {}", currentState);
break;
}
}
fileResource = fileQueue.poll(1L, TimeUnit.SECONDS);
if (fileResource != null) {
if (fileResource instanceof PoisonFileResource) {
setEndedState(STATE.SWALLOWED_POISON);
fileResource = null;
}
break;
}
LOG.debug("{} is waiting for file and the queue size is: {}", consumerId, fileQueue.size());
long elapsed = new Date().getTime() - start;
if (maxConsecWaitInMillis > 0 && elapsed > maxConsecWaitInMillis) {
setEndedState(STATE.EXCEEDED_MAX_CONSEC_WAIT_MILLIS);
break;
}
}
return fileResource;
}
protected void close(Closeable closeable) {
if (closeable != null) {
try {
closeable.close();
} catch (IOException e){
LOG.warn(e.getMessage(), e);
}
}
closeable = null;
}
protected void flushAndClose(Closeable closeable) {
if (closeable == null) {
return;
}
if (closeable instanceof Flushable){
try {
((Flushable)closeable).flush();
} catch (IOException e) {
LOG.warn(e.getMessage(), e);
}
}
close(closeable);
}
//do not overwrite a finished state except if
//not yet started, actively consuming or shutting down. This should
//represent the initial cause; all subsequent calls
//to set will be ignored!!!
private void setEndedState(STATE cause) {
synchronized(lock) {
if (currentState == STATE.NOT_YET_STARTED ||
currentState == STATE.ACTIVELY_CONSUMING ||
currentState == STATE.ASKED_TO_SHUTDOWN) {
currentState = cause;
}
}
}
/**
* Utility method to handle logging equivalently among all
* implementing classes. Use, override or avoid as desired.
*
* @param resourceId resourceId
* @param parser parser to use
* @param is inputStream (will be closed by this method!)
* @param handler handler for the content
* @param m metadata
* @param parseContext parse context
* @throws Throwable (logs and then throws whatever was thrown (if anything)
*/
protected void parse(final String resourceId, final Parser parser, InputStream is,
final ContentHandler handler,
final Metadata m, final ParseContext parseContext) throws Throwable {
try {
parser.parse(is, handler, m, parseContext);
} catch (Throwable t) {
if (t instanceof OutOfMemoryError) {
LOG.error(getXMLifiedLogMsg(OOM, resourceId, t));
} else if (t instanceof Error) {
LOG.error(getXMLifiedLogMsg(PARSE_ERR, resourceId, t));
} else {
LOG.warn(getXMLifiedLogMsg(PARSE_EX, resourceId, t));
incrementHandledExceptions();
}
throw t;
} finally {
close(is);
}
}
}