/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.TaskType;
class MemoryMatcher {
private static final Log LOG = LogFactory.getLog(MemoryMatcher.class);
private CapacityTaskScheduler scheduler;
public MemoryMatcher(CapacityTaskScheduler capacityTaskScheduler) {
this.scheduler = capacityTaskScheduler;
}
boolean isSchedulingBasedOnMemEnabled() {
if (scheduler.getLimitMaxMemForMapSlot()
== JobConf.DISABLED_MEMORY_LIMIT
|| scheduler.getLimitMaxMemForReduceSlot()
== JobConf.DISABLED_MEMORY_LIMIT
|| scheduler.getMemSizeForMapSlot()
== JobConf.DISABLED_MEMORY_LIMIT
|| scheduler.getMemSizeForReduceSlot()
== JobConf.DISABLED_MEMORY_LIMIT) {
return false;
}
return true;
}
/**
* Find the memory that is already used by all the running tasks
* residing on the given TaskTracker.
*
* @param taskTracker
* @param taskType
* @return amount of memory that is used by the residing tasks,
* null if memory cannot be computed for some reason.
*/
synchronized Long getMemReservedForTasks(
TaskTrackerStatus taskTracker, TaskType taskType) {
long vmem = 0;
for (TaskStatus task : taskTracker.getTaskReports()) {
// the following task states are one in which the slot is
// still occupied and hence memory of the task should be
// accounted in used memory.
if ((task.getRunState() == TaskStatus.State.RUNNING) ||
(task.getRunState() == TaskStatus.State.UNASSIGNED) ||
(task.inTaskCleanupPhase())) {
// Get the memory "allotted" for this task based on number of slots
long myVmem = 0;
if (task.getIsMap() && taskType == TaskType.MAP) {
long memSizePerMapSlot = scheduler.getMemSizeForMapSlot();
myVmem =
memSizePerMapSlot * task.getNumSlots();
} else if (!task.getIsMap()
&& taskType == TaskType.REDUCE) {
long memSizePerReduceSlot = scheduler.getMemSizeForReduceSlot();
myVmem = memSizePerReduceSlot * task.getNumSlots();
}
vmem += myVmem;
}
}
return Long.valueOf(vmem);
}
/**
* Check if a TT has enough memory to run of task specified from this job.
* @param job
* @param taskType
* @param taskTracker
* @return true if this TT has enough memory for this job. False otherwise.
*/
boolean matchesMemoryRequirements(JobInProgress job,TaskType taskType,
TaskTrackerStatus taskTracker) {
LOG.debug("Matching memory requirements of " + job.getJobID().toString()
+ " for scheduling on " + taskTracker.trackerName);
if (!isSchedulingBasedOnMemEnabled()) {
LOG.debug("Scheduling based on job's memory requirements is disabled."
+ " Ignoring any value set by job.");
return true;
}
Long memUsedOnTT = getMemReservedForTasks(taskTracker, taskType);
long totalMemUsableOnTT = 0;
long memForThisTask = 0;
if (taskType == TaskType.MAP) {
memForThisTask = job.getJobConf().getMemoryForMapTask();
totalMemUsableOnTT =
scheduler.getMemSizeForMapSlot() * taskTracker.getMaxMapSlots();
} else if (taskType == TaskType.REDUCE) {
memForThisTask = job.getJobConf().getMemoryForReduceTask();
totalMemUsableOnTT =
scheduler.getMemSizeForReduceSlot()
* taskTracker.getMaxReduceSlots();
}
long freeMemOnTT = totalMemUsableOnTT - memUsedOnTT.longValue();
if (memForThisTask > freeMemOnTT) {
if (LOG.isDebugEnabled()) {
LOG.debug("memForThisTask (" + memForThisTask + ") > freeMemOnTT ("
+ freeMemOnTT + "). A " + taskType + " task from "
+ job.getJobID().toString() + " cannot be scheduled on TT "
+ taskTracker.trackerName);
}
return false;
}
if (LOG.isDebugEnabled()) {
LOG.debug("memForThisTask = " + memForThisTask + ". freeMemOnTT = "
+ freeMemOnTT + ". A " + taskType.toString() + " task from "
+ job.getJobID().toString() + " matches memory requirements "
+ "on TT "+ taskTracker.trackerName);
}
return true;
}
}