/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import java.io.IOException;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.HashSet;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.Timer;
import java.util.TimerTask;
import java.util.Map;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
/**
* A {@link TaskScheduler} that
* provides the following features:
* (1) allows continuous enforcement of user controlled dynamic queue shares,
* (2) preempts tasks exceeding their queue shares instantaneously when new
* jobs arrive,
* (3) is work conserving,
* (4) tracks queue usage to only charge when jobs are pending or running,
* (5) authorizes queue submissions based on symmetric private key HMAC/SHA1
* signatures.
*/
class PriorityScheduler extends QueueTaskScheduler {
private class InitThread extends Thread {
JobInProgress job;
InitThread(JobInProgress job) {
this.job = job;
}
@Override
public void run() {
taskTrackerManager.initJob(job);
}
}
private class JobListener extends JobInProgressListener {
@Override
public void jobAdded(JobInProgress job) {
new InitThread(job).start();
synchronized (PriorityScheduler.this) {
String queue = authorize(job);
if (queue.equals("")) {
job.kill();
return;
}
jobQueue.add(job);
QueueJobs jobs = queueJobs.get(queue);
if (jobs == null) {
jobs = new QueueJobs(queue);
queueJobs.put(queue,jobs);
}
jobs.jobs.add(job);
if (debug) {
LOG.debug("Add job " + job.getProfile().getJobID());
}
}
}
@Override
public void jobRemoved(JobInProgress job) {
synchronized (PriorityScheduler.this) {
jobQueue.remove(job);
String queue = getQueue(job);
queueJobs.get(queue).jobs.remove(job);
}
}
@Override
public void jobUpdated(JobChangeEvent event) {
}
}
static final Comparator<TaskInProgress> TASK_COMPARATOR
= new Comparator<TaskInProgress>() {
public int compare(TaskInProgress o1, TaskInProgress o2) {
int res = 0;
if (o1.getProgress() < o2.getProgress()) {
res = -1;
} else {
res = (o1.getProgress() == o2.getProgress() ? 0 : 1);
}
if (res == 0) {
if (o1.getExecStartTime() > o2.getExecStartTime()) {
res = -1;
} else {
res = (o1.getExecStartTime() == o2.getExecStartTime() ? 0 : 1);
}
}
return res;
}
};
static final Comparator<KillQueue> QUEUE_COMPARATOR
= new Comparator<KillQueue>() {
public int compare(KillQueue o1, KillQueue o2) {
if (o1.startTime < o2.startTime) {
return 1;
}
if (o1.startTime > o2.startTime) {
return -1;
}
return 0;
}
};
class QueueJobs {
String name;
LinkedList<JobInProgress> jobs = new LinkedList<JobInProgress>();
QueueJobs(String name) {
this.name = name;
}
}
class QueueQuota {
int quota;
int map_used;
int reduce_used;
int map_pending;
int reduce_pending;
int mappers;
int reducers;
String name;
QueueQuota(String name) {
this.name = name;
}
}
private QueueAllocator allocator;
private static final Log LOG =
LogFactory.getLog(PriorityScheduler.class);
static final boolean MAP = true;
static final boolean REDUCE = false;
private static final boolean FILL = true;
private static final boolean NO_FILL = false;
private JobListener jobListener = new JobListener();
private static final boolean debug = LOG.isDebugEnabled();
private boolean sortTasks = true;
private long lastKill = 0;
private long killInterval = 0;
private PriorityAuthorization auth = new PriorityAuthorization();
private LinkedList<JobInProgress> jobQueue =
new LinkedList<JobInProgress>();
private HashMap<String,QueueJobs> queueJobs =
new HashMap<String,QueueJobs>();
@Override
public void start() throws IOException {
taskTrackerManager.addJobInProgressListener(jobListener);
sortTasks = conf.getBoolean("mapred.priority-scheduler.sort-tasks", true);
killInterval = conf.getLong("mapred.priority-scheduler.kill-interval", 0);
auth.init(conf);
}
@Override
public void terminate() throws IOException {
}
@Override
public void setAllocator(QueueAllocator allocator) {
this.allocator = allocator;
}
private boolean assignMapRedTask(JobInProgress job,
TaskTrackerStatus taskTracker, int numTrackers, List<Task> assignedTasks,
Map<String,QueueQuota> queueQuota, boolean fill, boolean map)
throws IOException {
String queue = getQueue(job);
QueueQuota quota = queueQuota.get(queue);
if (quota == null) {
LOG.error("Queue " + queue + " not configured properly");
return false;
}
if (quota.quota < 1 && !fill) {
return false;
}
Task t = null;
if (map) {
t = job.obtainNewLocalMapTask(taskTracker, numTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
if (debug) {
LOG.debug("assigned local task for job " + job.getProfile().getJobID() +
" " + taskType(map) );
}
assignedTasks.add(t);
if (map) {
quota.map_used++;
} else {
quota.reduce_used++;
}
quota.quota--;
return true;
}
t = job.obtainNewNonLocalMapTask(taskTracker, numTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
} else {
t = job.obtainNewReduceTask(taskTracker, numTrackers,
taskTrackerManager.getNumberOfUniqueHosts());
}
if (t != null) {
if (debug) {
LOG.debug("assigned remote task for job " + job.getProfile().getJobID() +
" " + taskType(map));
}
assignedTasks.add(t);
if (map) {
quota.map_used++;
} else {
quota.reduce_used++;
}
quota.quota--;
return true;
}
return false;
}
Map<String,QueueQuota> getQueueQuota(int maxMapTasks, int maxReduceTasks,
boolean map) {
if (debug) {
LOG.debug("max map tasks " + Integer.toString(maxMapTasks) + " " +
taskType(map));
LOG.debug("max reduce tasks " + Integer.toString(maxReduceTasks) + " " +
taskType(map));
}
int maxTasks = (map) ? maxMapTasks : maxReduceTasks;
Map<String,QueueAllocation> shares = allocator.getAllocation();
Map<String,QueueQuota> quotaMap = new HashMap<String,QueueQuota>();
for (QueueAllocation share: shares.values()) {
QueueQuota quota = new QueueQuota(share.getName());
quota.mappers = Math.round(share.getShare() * maxMapTasks);
quota.reducers = Math.round(share.getShare() * maxReduceTasks);
quota.quota = (map) ? quota.mappers : quota.reducers;
if (debug) {
LOG.debug("queue " + quota.name + " initial quota " +
Integer.toString(quota.quota) + " " + taskType(map));
}
quota.map_used = 0;
quota.reduce_used = 0;
quota.map_pending = 0;
quota.reduce_pending = 0;
Collection<JobInProgress> jobs = getJobs(quota.name);
for (JobInProgress job : jobs) {
quota.map_pending += job.pendingMaps();
quota.reduce_pending += job.pendingReduces();
int running = (map) ? job.runningMapTasks : job.runningReduceTasks;
quota.quota -= running;
quota.map_used += job.runningMapTasks ;
quota.reduce_used += job.runningReduceTasks;
}
if (debug) {
LOG.debug("queue " + quota.name + " quota " +
Integer.toString(quota.quota) + " " + taskType(map));
}
quotaMap.put(quota.name,quota);
}
return quotaMap;
}
private void scheduleJobs(int availableSlots, boolean map, boolean fill,
TaskTrackerStatus taskTracker, int numTrackers, List<Task> assignedTasks,
Map<String,QueueQuota> queueQuota) throws IOException {
for (int i = 0; i < availableSlots; i++) {
for (JobInProgress job : jobQueue) {
if ((job.getStatus().getRunState() != JobStatus.RUNNING) ||
(!map && job.numReduceTasks == 0)) {
continue;
}
if (assignMapRedTask(job, taskTracker, numTrackers, assignedTasks,
queueQuota, fill, map)) {
break;
}
}
}
}
private int countTasksToKill(Map<String,QueueQuota> queueQuota, boolean map) {
int killTasks = 0;
for (QueueQuota quota : queueQuota.values()) {
killTasks += Math.min((map) ? quota.map_pending : quota.reduce_pending,
Math.max(quota.quota,0));
}
return killTasks;
}
protected void markIdle(Map<String, QueueQuota> queueQuota) {
for (QueueQuota quota: queueQuota.values()) {
allocator.setUsage(quota.name, Math.min(quota.map_used, quota.mappers) +
Math.min(quota.reduce_used, quota.reducers),
(quota.map_pending + quota.reduce_pending));
}
}
private synchronized void assignMapRedTasks(List<Task> assignedTasks,
TaskTrackerStatus taskTracker, int numTrackers, boolean map)
throws IOException {
int taskOffset = assignedTasks.size();
int maxTasks = (map) ? taskTracker.getMaxMapSlots() :
taskTracker.getMaxReduceSlots();
int countTasks = (map) ? taskTracker.countMapTasks() :
taskTracker.countReduceTasks();
int availableSlots = maxTasks - countTasks;
int map_capacity = 0;
int reduce_capacity = 0;
ClusterStatus status = taskTrackerManager.getClusterStatus();
if (status != null) {
map_capacity = status.getMaxMapTasks();
reduce_capacity = status.getMaxReduceTasks();
}
Map<String,QueueQuota> queueQuota = getQueueQuota(map_capacity,
reduce_capacity,map);
if (debug) {
LOG.debug("available slots " + Integer.toString(availableSlots) + " " +
taskType(map));
LOG.debug("queue size " + Integer.toString(jobQueue.size()));
LOG.debug("map capacity " + Integer.toString(map_capacity) + " " +
taskType(map));
LOG.debug("reduce capacity " + Integer.toString(reduce_capacity) + " " +
taskType(map));
}
scheduleJobs(availableSlots, map, NO_FILL, taskTracker, numTrackers,
assignedTasks, queueQuota);
availableSlots -= assignedTasks.size() + taskOffset;
scheduleJobs(availableSlots, map, FILL, taskTracker, numTrackers,
assignedTasks, queueQuota);
if (map) {
markIdle(queueQuota);
}
long currentTime = System.currentTimeMillis()/1000;
if ((killInterval > 0) && (currentTime - lastKill > killInterval)) {
lastKill = currentTime;
} else {
return;
}
int killTasks = countTasksToKill(queueQuota, map);
if (debug) {
LOG.debug("trying to kill " + Integer.toString(killTasks) + " tasks " +
taskType(map));
}
killMapRedTasks(killTasks, queueQuota, map);
}
class KillQueue {
String name;
long startTime;
QueueQuota quota;
}
private Collection<KillQueue> getKillQueues(Map<String,
QueueQuota> queueQuota) {
TreeMap killQueues = new TreeMap(QUEUE_COMPARATOR);
for (QueueJobs queueJob : queueJobs.values()) {
QueueQuota quota = queueQuota.get(queueJob.name);
if (quota.quota >= 0) {
continue;
}
for (JobInProgress job : queueJob.jobs) {
if (job.getStatus().getRunState() == JobStatus.RUNNING) {
KillQueue killQueue = new KillQueue();
killQueue.name = queueJob.name;
killQueue.startTime = job.getStartTime();
killQueue.quota = quota;
killQueues.put(killQueue, killQueue);
}
}
}
return killQueues.values();
}
private void killMapRedTasks(int killTasks, Map<String,QueueQuota> queueQuota,
boolean map) {
int killed = 0;
// sort queues exceeding quota in reverse order of time since starting
// a running job
Collection<KillQueue> killQueues = getKillQueues(queueQuota);
for (KillQueue killQueue : killQueues) {
if (killed == killTasks) {
return;
}
QueueQuota quota = killQueue.quota;
// don't kill more than needed and not more than quota exceeded
int toKill = Math.min(killTasks-killed,-quota.quota);
killQueueTasks(quota.name, toKill, map);
killed += toKill;
}
}
private String taskType(boolean map) {
return (map) ? "MAP" : "REDUCE";
}
private void killQueueTasks(String queue, int killTasks, boolean map) {
if (killTasks == 0) {
return;
}
if (debug) {
LOG.debug("trying to kill " + Integer.toString(killTasks) +
" tasks from queue " + queue + " " + taskType(map));
}
int killed = 0;
Collection<JobInProgress> jobs = getJobs(queue);
if (debug) {
LOG.debug("total jobs to kill from " + Integer.toString(jobs.size()) +
" " + taskType(map));
}
for (JobInProgress job : jobs) {
TaskInProgress tasks[] = (map) ? job.maps.clone() :
job.reduces.clone();
if (sortTasks) {
Arrays.sort(tasks, TASK_COMPARATOR);
}
if (debug) {
LOG.debug("total tasks to kill from " +
Integer.toString(tasks.length) + " " + taskType(map));
}
for (int i=0; i < tasks.length; i++) {
if (debug) {
LOG.debug("total active tasks to kill from " +
Integer.toString(tasks[i].getActiveTasks().keySet().size()) +
" " + taskType(map));
}
for (TaskAttemptID id: tasks[i].getActiveTasks().keySet()) {
if (tasks[i].isCommitPending(id)) {
continue;
}
tasks[i].killTask(id, false);
if (debug) {
LOG.debug("killed task " + id + " progress " +
Double.toString(tasks[i].getProgress()) +
" start time " + Long.toString(tasks[i].getExecStartTime()) +
" " + taskType(map));
}
killed += 1;
if (killed == killTasks) {
return;
}
}
}
}
}
@Override
public List<Task> assignTasks(TaskTracker taskTracker)
throws IOException {
long millis = 0;
if (debug) {
millis = System.currentTimeMillis();
}
ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
int numTrackers = clusterStatus.getTaskTrackers();
List<Task> assignedTasks = new ArrayList<Task>();
assignMapRedTasks(assignedTasks, taskTracker.getStatus(), numTrackers, MAP);
assignMapRedTasks(assignedTasks, taskTracker.getStatus(), numTrackers, REDUCE);
if (debug) {
long elapsed = System.currentTimeMillis() - millis;
LOG.debug("assigned total tasks: " +
Integer.toString(assignedTasks.size()) + " in " +
Long.toString(elapsed) + " ms");
}
return assignedTasks;
}
@Override
public Collection<JobInProgress> getJobs(String queueName) {
QueueJobs jobs = queueJobs.get(queueName);
if (jobs == null) {
return new ArrayList<JobInProgress>();
}
return jobs.jobs;
}
private String getQueue(JobInProgress job) {
JobConf conf = job.getJobConf();
return conf.getQueueName();
}
private String getUser(JobInProgress job) {
JobConf conf = job.getJobConf();
return conf.getUser();
}
private String authorize(JobInProgress job) {
JobConf conf = job.getJobConf();
String user = conf.getUser();
String queue = conf.getQueueName();
if (!user.equals(queue)) {
return "";
}
String timestamp = conf.get("mapred.job.timestamp");
String signature = conf.get("mapred.job.signature");
int role = auth.authorize("&user=" + user + "×tamp=" + timestamp,
signature, user, timestamp);
if (role != PriorityAuthorization.NO_ACCESS) {
return queue;
}
return "";
}
}