/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.Map.Entry;
import javax.security.auth.login.LoginException;
import junit.extensions.TestSetup;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo;
import org.apache.hadoop.mapred.JobTracker.ReasonForBlackListing;
import org.apache.hadoop.mapred.TaskTrackerStatus.TaskTrackerHealthStatus;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
public class TestTaskTrackerBlacklisting extends TestCase {
static String trackers[] = new String[] { "tracker_tracker1:1000",
"tracker_tracker2:1000", "tracker_tracker3:1000" };
static String hosts[] = new String[] { "tracker1", "tracker2", "tracker3" };
private static FakeJobTracker jobTracker;
private static FakeJobTrackerClock clock;
private static short responseId;
private static final Set<ReasonForBlackListing> nodeUnHealthyReasonSet =
EnumSet.of(ReasonForBlackListing.NODE_UNHEALTHY);
private static final Set<ReasonForBlackListing> exceedsFailuresReasonSet =
EnumSet.of(ReasonForBlackListing.EXCEEDING_FAILURES);
private static final Set<ReasonForBlackListing>
unhealthyAndExceedsFailure = EnumSet.of(
ReasonForBlackListing.NODE_UNHEALTHY,
ReasonForBlackListing.EXCEEDING_FAILURES);
// Add extra millisecond where timer granularity is too coarse
private static final long aDay = 24 * 60 * 60 * 1000 + 1;
private static class FakeJobTrackerClock extends Clock {
boolean jumpADay = false;
@Override
long getTime() {
if (!jumpADay) {
return super.getTime();
} else {
long now = super.getTime();
return now + aDay;
}
}
}
static class FakeJobTracker extends
org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker {
FakeJobTracker(JobConf conf, Clock clock, String[] tts) throws IOException,
InterruptedException, LoginException {
super(conf, clock, tts);
}
@Override
synchronized void finalizeJob(JobInProgress job) {
List<String> blackListedTrackers = job.getBlackListedTrackers();
for (String tracker : blackListedTrackers) {
incrementFaults(tracker);
}
}
}
static class FakeJobInProgress extends
org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress {
HashMap<String, Integer> trackerToFailureMap;
FakeJobInProgress(JobConf jobConf, JobTracker tracker) throws IOException {
super(jobConf, tracker);
// initObjects(tracker, numMaps, numReduces);
trackerToFailureMap = new HashMap<String, Integer>();
}
public void failTask(TaskAttemptID taskId) {
super.failTask(taskId);
TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
addFailuresToTrackers(tip.machineWhereTaskRan(taskId));
}
public void addFailuresToTrackers(String trackerName) {
Integer numOfFailures = trackerToFailureMap.get(trackerName);
if (numOfFailures == null) {
numOfFailures = 0;
}
trackerToFailureMap.put(trackerName, numOfFailures + 1);
}
public List<String> getBlackListedTrackers() {
ArrayList<String> blackListedTrackers = new ArrayList<String>();
for (Entry<String, Integer> entry : trackerToFailureMap.entrySet()) {
Integer failures = entry.getValue();
String tracker = entry.getKey();
if (failures.intValue() >= this.getJobConf()
.getMaxTaskFailuresPerTracker()) {
blackListedTrackers.add(JobInProgress
.convertTrackerNameToHostName(tracker));
}
}
return blackListedTrackers;
}
}
public static Test suite() {
TestSetup setup =
new TestSetup(new TestSuite(TestTaskTrackerBlacklisting.class)) {
protected void setUp() throws Exception {
JobConf conf = new JobConf();
conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0");
conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0");
conf.setInt(JTConfig.JT_MAX_TRACKER_BLACKLISTS, 1);
jobTracker =
new FakeJobTracker(conf, (clock = new FakeJobTrackerClock()),
trackers);
sendHeartBeat(null, true);
}
protected void tearDown() throws Exception {
//delete the build/test/logs/ dir
}
};
return setup;
}
private static void sendHeartBeat(TaskTrackerHealthStatus status,
boolean initialContact)
throws IOException {
for (String tracker : trackers) {
TaskTrackerStatus tts = new TaskTrackerStatus(tracker, JobInProgress
.convertTrackerNameToHostName(tracker));
if (status != null) {
TaskTrackerHealthStatus healthStatus = tts.getHealthStatus();
healthStatus.setNodeHealthy(status.isNodeHealthy());
healthStatus.setHealthReport(status.getHealthReport());
healthStatus.setLastReported(status.getLastReported());
}
jobTracker.heartbeat(tts, false, initialContact,
false, responseId);
}
responseId++;
}
public void testTrackerBlacklistingForJobFailures() throws Exception {
runBlackListingJob(jobTracker, trackers);
assertEquals("Tracker 1 not blacklisted", jobTracker
.getBlacklistedTrackerCount(), 1);
checkReasonForBlackListing(hosts[0], exceedsFailuresReasonSet);
clock.jumpADay = true;
sendHeartBeat(null, false);
assertEquals("Tracker 1 still blacklisted after a day", 0, jobTracker
.getBlacklistedTrackerCount());
//Cleanup the blacklisted trackers.
//Tracker is black listed due to failure count, so clock has to be
//forwarded by a day.
clock.jumpADay = false;
}
public void testNodeHealthBlackListing() throws Exception {
TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR");
//Blacklist tracker due to node health failures.
sendHeartBeat(status, false);
for (String host : hosts) {
checkReasonForBlackListing(host, nodeUnHealthyReasonSet);
}
status.setNodeHealthy(true);
status.setLastReported(System.currentTimeMillis());
status.setHealthReport("");
//white list tracker so the further test cases can be
//using trackers.
sendHeartBeat(status, false);
assertEquals("Trackers still blacklisted after healthy report", 0,
jobTracker.getBlacklistedTrackerCount());
}
/**
* Test case to check if the task tracker node health failure statistics
* is populated correctly.
*
* We check the since start property and assume that other properties would
* be populated in a correct manner.
*/
public void testTaskTrackerNodeHealthFailureStatistics() throws Exception {
//populate previous failure count, as the job tracker is bought up only
//once in setup of test cases to run all node health blacklist stuff.
int failureCount = getFailureCountSinceStart(jobTracker, trackers[0]);
sendHeartBeat(null, false);
for(String tracker: trackers) {
assertEquals("Failure count updated wrongly for tracker : " + tracker,
failureCount, getFailureCountSinceStart(jobTracker, tracker));
}
TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR");
sendHeartBeat(status, false);
//When the node fails due to health check, the statistics is
//incremented.
failureCount++;
for(String tracker: trackers) {
assertEquals("Failure count updated wrongly for tracker : " + tracker,
failureCount, getFailureCountSinceStart(jobTracker, tracker));
}
//even if the node reports unhealthy in next status update we dont
//increment it. We increment the statistics if the node goes back to
//healthy and then becomes unhealthy.
sendHeartBeat(status, false);
for(String tracker: trackers) {
assertEquals("Failure count updated wrongly for tracker : " + tracker,
failureCount, getFailureCountSinceStart(jobTracker, tracker));
}
//make nodes all healthy, but the failure statistics should be
//carried forward.
sendHeartBeat(null, false);
for(String tracker: trackers) {
assertEquals("Failure count updated wrongly for tracker : " + tracker,
failureCount, getFailureCountSinceStart(jobTracker, tracker));
}
}
private int getFailureCountSinceStart(JobTracker jt, String tracker) {
JobTrackerStatistics jtStats = jt.getStatistics();
StatisticsCollector collector = jtStats.collector;
collector.update();
return jtStats.getTaskTrackerStat(tracker).healthCheckFailedStat
.getValues().get(StatisticsCollector.SINCE_START).getValue();
}
public void testBlackListingWithFailuresAndHealthStatus() throws Exception {
runBlackListingJob(jobTracker, trackers);
assertEquals("Tracker 1 not blacklisted", 1,
jobTracker.getBlacklistedTrackerCount());
checkReasonForBlackListing(hosts[0], exceedsFailuresReasonSet);
TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR");
sendHeartBeat(status, false);
assertEquals("All trackers not blacklisted", 3,
jobTracker.getBlacklistedTrackerCount());
checkReasonForBlackListing(hosts[0], unhealthyAndExceedsFailure);
checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet);
checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet);
clock.jumpADay = true;
sendHeartBeat(status, false);
assertEquals("All trackers not blacklisted", 3,
jobTracker.getBlacklistedTrackerCount());
for (String host : hosts) {
checkReasonForBlackListing(host, nodeUnHealthyReasonSet);
}
//clear blacklisted trackers due to node health reasons.
sendHeartBeat(null, false);
assertEquals("All trackers not white listed", 0,
jobTracker.getBlacklistedTrackerCount());
//Clear the blacklisted trackers due to failures.
clock.jumpADay = false;
}
public void testBlacklistingReasonString() throws Exception {
String error = "ERROR";
String error1 = "ERROR1";
TaskTrackerHealthStatus status = getUnhealthyNodeStatus(error);
sendHeartBeat(status, false);
assertEquals("All trackers not blacklisted", 3,
jobTracker.getBlacklistedTrackerCount());
checkReasonForBlackListing(hosts[0], nodeUnHealthyReasonSet);
checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet);
checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet);
for (int i = 0; i < hosts.length; i++) {
//Replace new line as we are adding new line
//in getFaultReport
assertEquals("Blacklisting reason string not correct for host " + i,
error,
jobTracker.getFaultReport(hosts[i]).replace("\n", ""));
}
status.setNodeHealthy(false);
status.setLastReported(System.currentTimeMillis());
status.setHealthReport(error1);
sendHeartBeat(status, false);
checkReasonForBlackListing(hosts[0], nodeUnHealthyReasonSet);
checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet);
checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet);
for (int i = 0; i < hosts.length; i++) {
//Replace new line as we are adding new line
//in getFaultReport
assertEquals("Blacklisting reason string not correct for host " + i,
error1,
jobTracker.getFaultReport(hosts[i]).replace("\n", ""));
}
//clear the blacklisted trackers with node health reasons.
sendHeartBeat(null, false);
}
private TaskTrackerHealthStatus getUnhealthyNodeStatus(String error) {
TaskTrackerHealthStatus status = new TaskTrackerHealthStatus();
status.setNodeHealthy(false);
status.setLastReported(System.currentTimeMillis());
status.setHealthReport(error);
return status;
}
public void testBlackListingWithTrackerReservation() throws Exception {
JobConf conf = new JobConf();
conf.setNumMapTasks(1);
conf.setNumReduceTasks(1);
FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]);
TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
tt1.reserveSlots(TaskType.MAP, job, 1);
tt1.reserveSlots(TaskType.REDUCE, job, 1);
tt2.reserveSlots(TaskType.MAP, job, 1);
tt2.reserveSlots(TaskType.REDUCE, job, 1);
assertEquals("Tracker 1 not reserved for the job 1", 2, job
.getNumReservedTaskTrackersForMaps());
assertEquals("Tracker 1 not reserved for the job 1", 2, job
.getNumReservedTaskTrackersForReduces());
runBlackListingJob(jobTracker, trackers);
assertEquals("Tracker 1 not unreserved for the job 1", 1, job
.getNumReservedTaskTrackersForMaps());
assertEquals("Tracker 1 not unreserved for the job 1", 1, job
.getNumReservedTaskTrackersForReduces());
assertEquals("Tracker 1 not blacklisted", 1, jobTracker
.getBlacklistedTrackerCount());
checkReasonForBlackListing(hosts[0], exceedsFailuresReasonSet);
TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR");
sendHeartBeat(status, false);
assertEquals("All trackers not blacklisted", 3,
jobTracker.getBlacklistedTrackerCount());
checkReasonForBlackListing(hosts[0], unhealthyAndExceedsFailure);
checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet);
checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet);
assertEquals("Tracker 1 not unreserved for the job 1", 0, job
.getNumReservedTaskTrackersForMaps());
assertEquals("Tracker 1 not unreserved for the job 1", 0, job
.getNumReservedTaskTrackersForReduces());
//white list all trackers for health reasons and failure counts
clock.jumpADay = true;
sendHeartBeat(null, false);
}
/**
* Test case to test if the cluster status is populated with the right
* blacklist information, which would be used by the {@link JobClient} to
* display information on the Command Line interface.
*
*/
public void testClusterStatusBlacklistedReason() throws Exception {
String error = "ERROR";
String errorWithNewLines = "ERROR\nERROR";
String expectedErrorReport = "ERROR:ERROR";
// Create an unhealthy tracker health status.
Collection<BlackListInfo> blackListedTrackerInfo = jobTracker
.getBlackListedTrackers();
assertTrue("The blacklisted tracker nodes is not empty.",
blackListedTrackerInfo.isEmpty());
TaskTrackerHealthStatus status = getUnhealthyNodeStatus(errorWithNewLines);
// make all tracker unhealthy
sendHeartBeat(status, false);
assertEquals("All trackers not blacklisted", 3, jobTracker
.getBlacklistedTrackerCount());
// Verify the new method .getBlackListedTracker() which is
// used by the ClusterStatus to set the list of blacklisted
// tracker.
blackListedTrackerInfo = jobTracker.getBlackListedTrackers();
// Check if all the black listed tracker information is obtained
// in new method.
assertEquals("Blacklist tracker info does not contain all trackers", 3,
blackListedTrackerInfo.size());
// verify all the trackers are blacklisted for health reasons.
// Also check the health report.
for (BlackListInfo bi : blackListedTrackerInfo) {
assertEquals("Tracker not blacklisted for health reason",
ReasonForBlackListing.NODE_UNHEALTHY.toString().trim(), bi
.getReasonForBlackListing().trim());
assertTrue("Tracker blacklist report does not match",
bi.toString().endsWith(expectedErrorReport));
}
// reset the tracker health status back to normal.
sendHeartBeat(null, false);
runBlackListingJob(jobTracker, trackers);
sendHeartBeat(status, false);
blackListedTrackerInfo = jobTracker.getBlackListedTrackers();
for (BlackListInfo bi : blackListedTrackerInfo) {
if (bi.getTrackerName().equals(trackers[0])) {
assertTrue(
"Reason for blacklisting of tracker 1 does not contain Unhealthy reasons",
bi.getReasonForBlackListing().contains(
ReasonForBlackListing.NODE_UNHEALTHY.toString().trim()));
assertTrue(
"Reason for blacklisting of tracker 1 does not contain Unhealthy reasons",
bi.getReasonForBlackListing().contains(
ReasonForBlackListing.EXCEEDING_FAILURES.toString().trim()));
assertTrue("Blacklist failure does not contain failure report string",
bi.getBlackListReport().contains("failures on the tracker"));
} else {
assertEquals("Tracker not blacklisted for health reason",
ReasonForBlackListing.NODE_UNHEALTHY.toString().trim(), bi
.getReasonForBlackListing().trim());
}
assertTrue("Tracker blacklist report does not match", bi
.getBlackListReport().trim().contains(error));
}
clock.jumpADay = true;
sendHeartBeat(null, false);
}
/**
* Runs a job which blacklists the first of the tracker
* which is passed to the method.
*
* @param jobTracker JobTracker instance
* @param trackers array of trackers, the method would blacklist
* first element of the array
* @return A job in progress object.
* @throws Exception
*/
static FakeJobInProgress runBlackListingJob(JobTracker jobTracker,
String[] trackers) throws Exception {
TaskAttemptID[] taskAttemptID = new TaskAttemptID[3];
JobConf conf = new JobConf();
conf.setSpeculativeExecution(false);
conf.setNumMapTasks(0);
conf.setNumReduceTasks(5);
conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70");
conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false);
conf.setMaxTaskFailuresPerTracker(1);
FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
job.setClusterSize(trackers.length);
job.initTasks();
taskAttemptID[0] = job.findReduceTask(trackers[0]);
taskAttemptID[1] = job.findReduceTask(trackers[1]);
taskAttemptID[2] = job.findReduceTask(trackers[2]);
job.finishTask(taskAttemptID[1]);
job.finishTask(taskAttemptID[2]);
job.failTask(taskAttemptID[0]);
taskAttemptID[0] = job.findReduceTask(trackers[0]);
job.failTask(taskAttemptID[0]);
taskAttemptID[0] = job.findReduceTask(trackers[1]);
job.finishTask(taskAttemptID[0]);
taskAttemptID[0] = job.findReduceTask(trackers[1]);
taskAttemptID[1] = job.findReduceTask(trackers[2]);
job.finishTask(taskAttemptID[0]);
job.finishTask(taskAttemptID[1]);
jobTracker.finalizeJob(job);
return job;
}
private void checkReasonForBlackListing(String host,
Set<ReasonForBlackListing> reasonsForBlackListing) {
Set<ReasonForBlackListing> rfbs = jobTracker.getReasonForBlackList(host);
assertEquals("Reasons for blacklisting of " + host + " does not match",
reasonsForBlackListing, rfbs);
}
}