// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import com.google.common.annotations.VisibleForTesting;
import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
import com.google.devtools.build.lib.util.Preconditions;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* An object that can monitor whether actions are getting completed in a timely manner.
*
* <p>If there's nothing happening for a while, a background thread will print (and update) the
* "Still waiting for N actions to complete..." message.
*/
public final class ActionExecutionInactivityWatchdog {
/** An object used in monitoring action execution inactivity. */
public interface InactivityMonitor {
/** Returns whether action execution has started. */
boolean hasStarted();
/** Returns the number of enqueued but not yet completed actions. */
int getPending();
/**
* Waits for any action to complete, or the timeout to elapse.
*
* <p>The thread must wait at least for the specified timeout, unless some action completes in
* the meantime. It's not allowed to return 0 too early.
*
* <p>Note that it's acceptable to return (any value) later than specified by the timeout.
*
* @return the number of actions completed during the wait
*/
int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException;
}
/** An object that the watchdog can report inactivity to. */
public interface InactivityReporter {
/**
* Report that actions are not getting completed in a timely manner.
*
* <p>Inactivity is typically not reported if tests with streaming output are being run.
*/
void maybeReportInactivity();
}
@VisibleForTesting
interface Sleep {
void sleep(int durationMilliseconds) throws InterruptedException;
}
private static final class WaitTime {
private final int progressIntervalFlagValue;
private int prev;
public WaitTime(int progressIntervalFlagValue) {
this.progressIntervalFlagValue = progressIntervalFlagValue;
}
public void reset() {
prev = 0;
}
public int next() {
prev = ActionExecutionStatusReporter.getWaitTime(progressIntervalFlagValue, prev);
return prev;
}
}
private final AtomicBoolean isRunning = new AtomicBoolean(false);
private final InactivityMonitor monitor;
private final InactivityReporter reporter;
private final Sleep sleeper;
private final Thread thread;
private final WaitTime waitTime;
public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter,
int progressIntervalFlagValue) {
this(monitor, reporter, progressIntervalFlagValue, new Sleep() {
@Override
public void sleep(int durationMilliseconds) throws InterruptedException {
Thread.sleep(durationMilliseconds);
}
});
}
@VisibleForTesting
public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter,
int progressIntervalFlagValue, Sleep sleeper) {
this.monitor = Preconditions.checkNotNull(monitor);
this.reporter = Preconditions.checkNotNull(reporter);
this.sleeper = Preconditions.checkNotNull(sleeper);
this.waitTime = new WaitTime(progressIntervalFlagValue);
this.thread = new Thread(new Runnable() {
@Override
public void run() {
enterWatchdogLoop();
}
});
this.thread.setDaemon(true);
this.thread.setName("action-execution-watchdog");
}
/** Starts the watchdog thread. This method should only be called once. */
public void start() {
Preconditions.checkState(!isRunning.getAndSet(true));
thread.start();
}
/**
* Stops the watchdog thread. This method should only be called once.
*
* <p>The method waits for the thread to terminate. If the caller thread is interrupted
* in the meantime, the interrupted status will be set.
*/
public void stop() {
Preconditions.checkState(isRunning.getAndSet(false));
thread.interrupt();
try {
thread.join();
} catch (InterruptedException e) {
// When Thread.join throws, the interrupted status is cleared. We need to set it again.
Thread.currentThread().interrupt();
}
}
private void enterWatchdogLoop() {
while (isRunning.get()) {
try {
// Wait a while for any SkyFunction to finish. The returned number indicates how many
// actions completed during the wait. It's possible that this is more than 1, since
// this thread may not immediately regain control.
int completedActions = monitor.waitForNextCompletion(waitTime.next() * 1000);
if (!isRunning.get()) {
break;
}
int pending = monitor.getPending();
if (!monitor.hasStarted() || completedActions > 0 || pending == 0) {
// If no keys have been enqueued yet (execution hasn't started), or some actions
// were completed since this thread was notified (we are making visible progress),
// or there are currently no enqueued actions waiting to be processed (perhaps all
// have completed and we are about to stop monitoring), then there's no need to
// display any messages.
waitTime.reset();
// Sleep a while before checking again. Actions might be executing at a nice rate, no
// need to worry about inactivity. This extra sleep isn't required but it's nice to
// have: without it we would, at times of high action completion rate, unnecessarily
// put the monitor into a fast sleep-wake cycle --- not a big problem but wasteful.
sleeper.sleep(1000);
} else {
// If actions are executing but we haven't made any progress in a while (no new
// action completion), then reassure the user that we're still running. Next time
// wait a little longer.
reporter.maybeReportInactivity();
}
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
return;
}
}
}
}