// Copyright 2009 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.enterprise.connector.instantiator;
import com.google.enterprise.connector.util.Clock;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Pool for running {@link TimedCancelable}, time limited tasks.
* <p>
* Users are provided a {@link TaskHandle} for each task. The {@link TaskHandle}
* supports canceling the task and determining if the task is done running.
* <p>
* The ThreadPool enforces a configurable maximum time interval for tasks. Each
* task is guarded by a <b>time out task</b> that will cancel the primary task
* if the primary task does not complete within the allowed interval.
* <p>
* If the configured maximum time interval is zero, tasks are allowed to run
* until explicitly cancelled, or shutdown.
* <p>
* Task cancellation includes two actions that are visible for the task task's
* {@link TimedCancelable}
* <ol>
* <li>Calling {@link Future#cancel(boolean)} to send the task an interrupt and
* mark it as done.</li>
* <li>Calling {@link TimedCancelable#cancel()} to send the task a second signal
* that it is being canceled. This signal has the benefit that it does not
* depend on the tasks interrupt handling policy.</li>
* </ol>
* Once a task has been canceled its {@link TaskHandle#isDone()} method will
* immediately start returning {@code true}.
* <p>
* {@link ThreadPool} performs the following processing when a task completes
* <ol>
* <li>Cancel the <b>time out task</b> for the completed task.</li>
* <li>Log exceptions that indicate the task did not complete normally.</li>
* </ol>
*/
/* This class is a thin wrapper around a lazily constructed instance of a
* LazyThreadPool implementation. This was done to avoid Tomcat shutdown
* hangs after Spring application Context initialization failures.
*/
public class ThreadPool {
private static final Logger LOGGER =
Logger.getLogger(ThreadPool.class.getName());
/**
* The default amount of time in to wait for tasks to complete during during
* shutdown.
*/
public static final int DEFAULT_SHUTDOWN_TIMEOUT_MILLIS = 10 * 1000;
/**
* Configured amount of time to let tasks run before automatic cancellation.
*/
private final long maximumTaskLifeMillis;
/**
* Clock used to time out threads.
*/
private final Clock clock;
/**
* Flag indicating shutdown was called. Don't spawn new tasks even if asked.
*/
private boolean isShutdown = false;
/**
* The lazily constructed LazyThreadPool instance.
*/
private LazyThreadPool lazyThreadPool;
/**
* Create a {@link ThreadPool}.
*
* @param taskLifeSeconds minimum number of seconds to allow a task to run
* before automatic cancellation. If zero, tasks will not time out.
* @param a {@link Clock} that is used to time tasks.
*/
// TODO: This method, called from Spring, multiplies the supplied [soft]
// timeout value by 2. The actual value wants to be 2x or 1.5x of a user
// configured soft value. However, Spring v2 does not provide a convenient
// mechanism to do arithmetic on configuration properties. Once we move to
// Spring v3, the calculation should be done in the Spring XML definition
// file rather than here.
public ThreadPool(int taskLifeSeconds, Clock clock) {
this.maximumTaskLifeMillis = taskLifeSeconds * 2 * 1000L;
this.clock = clock;
}
/**
* Shut down the {@link ThreadPool}. After this returns
* {@link ThreadPool#submit(TimedCancelable)} will return null.
*
* @param interrupt {@code true} if the threads executing tasks task should
* be interrupted; otherwise, in-progress tasks are allowed to complete
* normally.
* @param waitMillis maximum amount of time to wait for tasks to complete.
* @return {@code true} if all the running tasks terminated and
* {@code false} if the some running task did not terminate.
* @throws InterruptedException if interrupted while waiting.
*/
synchronized boolean shutdown(boolean interrupt, long waitMillis)
throws InterruptedException {
isShutdown = true;
if (lazyThreadPool == null) {
return true;
} else {
return lazyThreadPool.shutdown(interrupt, waitMillis);
}
}
/**
* Return a LazyThreadPool.
*/
private synchronized LazyThreadPool getInstance() {
if (lazyThreadPool == null) {
lazyThreadPool = new LazyThreadPool();
}
return lazyThreadPool;
}
/**
* Submit a {@link Cancelable} for execution and return a
* {@link TaskHandle} for the running task or null if the task has not been
* accepted. After {@link ThreadPool#shutdown(boolean, long)} returns this
* will always return null.
*/
public TaskHandle submit(Cancelable cancelable) {
if (isShutdown) {
return null;
}
if (cancelable instanceof TimedCancelable && maximumTaskLifeMillis != 0L) {
return getInstance().submit((TimedCancelable) cancelable);
} else {
return getInstance().submit(cancelable);
}
}
/**
* Lazily constructed ThreadPool implementation.
*/
private class LazyThreadPool {
/**
* ExecutorService for running submitted tasks. Tasks are only submitted
* through completionService.
*/
private final ExecutorService executor;
/**
* CompletionService for running submitted tasks. All tasks are submitted
* through this CompletionService to provide blocking, queued access to
* completion information.
*/
private final CompletionService<?> completionService;
/**
* Dedicated ExecutorService for running the CompletionTask. The completion
* task is run in its own ExecutorService so that it can be shut down after
* the executor for submitted tasks has been shut down and drained of
* running tasks.
*/
private final ExecutorService completionExecutor;
/**
* Dedicated ScheduledThreadPoolExecutor for running time out tasks. Each
* primary task is guarded by a time out task that is scheduled to run when
* the primary tasks maximum life time expires. When the time out task runs
* it cancels the primary task.
*/
private final ScheduledThreadPoolExecutor timeoutService;
LazyThreadPool() {
executor = Executors.newCachedThreadPool(
new ThreadNamingThreadFactory("ThreadPoolExecutor"));
completionService = new ExecutorCompletionService<Object>(executor);
completionExecutor = Executors.newSingleThreadExecutor(
new ThreadNamingThreadFactory("ThreadPoolCompletion"));
if (maximumTaskLifeMillis != 0L) {
timeoutService = new ScheduledThreadPoolExecutor(1,
new ThreadNamingThreadFactory("ThreadPoolTimeout"));
} else {
timeoutService = null;
}
completionExecutor.execute(new CompletionTask());
}
/**
* Shut down the LazyThreadPool.
* @param interrupt {@code true} if the threads executing tasks task should
* be interrupted; otherwise, in-progress tasks are allowed to
* complete normally.
* @param waitMillis maximum amount of time to wait for tasks to complete.
* @return {@code true} if all the running tasks terminated, or
* {@code false} if some running task did not terminate.
* @throws InterruptedException if interrupted while waiting.
*/
boolean shutdown(boolean interrupt, long waitMillis)
throws InterruptedException {
if (interrupt) {
executor.shutdownNow();
} else {
executor.shutdown();
}
if (timeoutService != null) {
timeoutService.shutdown();
}
try {
return executor.awaitTermination(waitMillis, TimeUnit.MILLISECONDS);
} finally {
completionExecutor.shutdownNow();
if (timeoutService != null) {
timeoutService.shutdownNow();
}
}
}
/**
* Submit a {@link TimedCancelable} for execution and return a
* {@link TaskHandle} for the running task or null if the task has not been
* accepted. After {@link LazyThreadPool#shutdown(boolean, long)} returns
* this will always return null.
*/
TaskHandle submit(TimedCancelable cancelable) {
try {
// When timeoutTask is run it will cancel 'cancelable'.
TimeoutTask timeoutTask = new TimeoutTask(cancelable);
// Schedule timeoutTask to run when 'cancelable's maximum run interval
// has expired.
// timeoutFuture will be used to cancel timeoutTask when 'cancelable'
// completes.
Future<?> timeoutFuture = timeoutService.schedule(timeoutTask,
maximumTaskLifeMillis, TimeUnit.MILLISECONDS);
// cancelTimeoutRunnable runs 'cancelable'. When 'cancelable' completes
// cancelTimeoutRunnable cancels 'timeoutTask'. This saves system
// resources. In addition it prevents timeout task from running and
// calling cancel after 'cancelable' completes successfully.
CancelTimeoutRunnable cancelTimeoutRunnable =
new CancelTimeoutRunnable(cancelable, timeoutFuture);
// taskFuture is used to cancel 'cancelable' and to determine if
// 'cancelable' is done.
Future<?> taskFuture =
completionService.submit(cancelTimeoutRunnable, null);
TaskHandle handle =
new TaskHandle(cancelable, taskFuture, clock.getTimeMillis());
// TODO(strellis): test/handle timer pop/cancel before submit. In
// production with a 30 minute timeout this should never happen.
timeoutTask.setTaskHandle(handle);
return handle;
} catch (RejectedExecutionException re) {
if (!executor.isShutdown()) {
LOGGER.log(Level.SEVERE, "Unable to execute task", re);
}
return null;
}
}
/**
* Submit a {@link Cancelable} for execution and return a
* {@link TaskHandle} for the running task or null if the task has not been
* accepted. After {@link LazyThreadPool#shutdown(boolean, long)} returns
* this will always return null.
*/
TaskHandle submit(Cancelable cancelable) {
try {
// taskFuture is used to cancel 'cancelable' and to determine if
// 'cancelable' is done.
Future<?> taskFuture = completionService.submit(cancelable, null);
return new TaskHandle(cancelable, taskFuture, clock.getTimeMillis());
} catch (RejectedExecutionException re) {
if (!executor.isShutdown()) {
LOGGER.log(Level.SEVERE, "Unable to execute task", re);
}
return null;
}
}
/**
* A {@link Runnable} for running {@link TimedCancelable} that has been
* guarded by a timeout task. This will cancel the timeout task when the
* {@link TimedCancelable} completes. If the timeout task has already run,
* then canceling it has no effect.
*/
private class CancelTimeoutRunnable implements Runnable {
private final Future<?> timeoutFuture;
private final TimedCancelable cancelable;
/**
* Constructs a {@link CancelTimeoutRunnable}.
*
* @param cancelable the {@link TimedCancelable} this runs.
* @param timeoutFuture the {@link Future} for canceling the timeout task.
*/
CancelTimeoutRunnable(TimedCancelable cancelable, Future<?> timeoutFuture) {
this.timeoutFuture = timeoutFuture;
this.cancelable = cancelable;
}
public void run() {
try {
cancelable.run();
} finally {
timeoutFuture.cancel(true);
timeoutService.purge();
}
}
}
/**
* A task that gets completion information from all the tasks that run in a
* {@link CompletionService} and logs uncaught exceptions that cause the
* tasks to fail.
*/
private class CompletionTask implements Runnable {
private void completeTask() throws InterruptedException {
Future<?> future = completionService.take();
try {
future.get();
} catch (CancellationException e) {
LOGGER.info("Batch terminated due to cancellation.");
} catch (ExecutionException e) {
Throwable cause = e.getCause();
// TODO(strellis): Should we call cancelable.cancel() if we get an
// exception?
if (cause instanceof InterruptedException) {
LOGGER.log(Level.INFO, "Batch terminated due to an interrupt.",
cause);
} else {
LOGGER.log(Level.SEVERE, "Batch failed with unhandled exception: ",
cause);
}
}
}
public void run() {
try {
while (!Thread.currentThread().isInterrupted()) {
completeTask();
}
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
LOGGER.info("Completion task shutdown.");
}
}
}
/**
* A task that cancels another task that is running a {@link TimedCancelable}.
* The {@link TimeoutTask} should be scheduled to run when the interval for
* the {@link TimedCancelable} to run expires.
*/
private static class TimeoutTask implements Runnable {
final TimedCancelable timedCancelable;
private volatile TaskHandle taskHandle;
TimeoutTask(TimedCancelable timedCancelable) {
this.timedCancelable = timedCancelable;
}
public void run() {
if (taskHandle != null) {
timedCancelable.timeout(taskHandle);
}
}
void setTaskHandle(TaskHandle taskHandle) {
this.taskHandle = taskHandle;
}
}
/**
* A {@link ThreadFactory} that adds a prefix to thread names assigned
* by {@link Executors#defaultThreadFactory()} to provide diagnostic
* context in stack traces.
*/
private static class ThreadNamingThreadFactory implements ThreadFactory {
private final ThreadFactory delegate = Executors.defaultThreadFactory();
private final String namePrefix;
ThreadNamingThreadFactory(String namePrefix) {
this.namePrefix = namePrefix + "-";
}
public Thread newThread(Runnable r) {
Thread t = delegate.newThread(r);
t.setName(namePrefix + t.getName());
return t;
}
}
}