/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import junit.framework.TestCase;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
import java.io.IOException;
import static org.apache.hadoop.mapred.CapacityTestUtils.*;
public class TestContainerQueue extends TestCase {
CapacityTestUtils.FakeTaskTrackerManager taskTrackerManager = null;
CapacityTaskScheduler scheduler = null;
CapacityTestUtils.FakeClock clock = null;
CapacityTestUtils.ControlledInitializationPoller
controlledInitializationPoller;
@Override
protected void setUp() {
setUp(2, 2, 1);
}
private void setUp(
int numTaskTrackers, int numMapTasksPerTracker,
int numReduceTasksPerTracker) {
taskTrackerManager =
new CapacityTestUtils.FakeTaskTrackerManager(
numTaskTrackers, numMapTasksPerTracker,
numReduceTasksPerTracker);
clock = new CapacityTestUtils.FakeClock();
scheduler = new CapacityTaskScheduler(clock);
scheduler.setTaskTrackerManager(taskTrackerManager);
// Don't let the JobInitializationPoller come in our way.
controlledInitializationPoller =
new CapacityTestUtils.ControlledInitializationPoller(
scheduler.jobQueuesManager, taskTrackerManager);
scheduler.setInitializationPoller(controlledInitializationPoller);
scheduler.setConf(taskTrackerManager.defaultJobConf);
}
/**
* check the minCapacity distribution across children
* and grand children.
* <p/>
* Check for both capacity % and also actual no of slots allocated.
*/
public void testMinCapacity() {
AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
//Simple check to make sure that capacity is properly distributed among
// its children.
//level 1 children
QueueSchedulingContext a1 = new QueueSchedulingContext(
"a", 25, -1, -1);
QueueSchedulingContext a2 = new QueueSchedulingContext(
"b", 25, -1, -1);
AbstractQueue q = new ContainerQueue(rt, a1);
AbstractQueue ql = new ContainerQueue(rt, a2);
//level 2 children
QueueSchedulingContext a = new QueueSchedulingContext(
"aa", 50, -1, -1);
QueueSchedulingContext b = new QueueSchedulingContext(
"ab", 50, -1, -1);
QueueSchedulingContext c = new QueueSchedulingContext(
"ac", 50, -1, -1);
QueueSchedulingContext d = new QueueSchedulingContext(
"ad", 50, -1, -1);
AbstractQueue q1 = new JobQueue(q, a);
AbstractQueue q2 = new JobQueue(q, b);
AbstractQueue q3 = new JobQueue(ql, c);
AbstractQueue q4 = new JobQueue(ql, d);
rt.update(1000, 1000);
//share at level 0.
// (1000 * 25) / 100
assertEquals(q.getQueueSchedulingContext().getMapTSC().getCapacity(), 250);
assertEquals(ql.getQueueSchedulingContext().getMapTSC().getCapacity(), 250);
//share would be (1000 * 25 / 100 ) * (50 / 100)
assertEquals(q1.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
assertEquals(q2.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
assertEquals(q3.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
assertEquals(q4.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
//
rt.update(1, 1);
assertEquals(q.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
assertEquals(ql.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
//share would be (1000 * 25 / 100 ) * (50 / 100)
assertEquals(q1.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
assertEquals(q2.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
assertEquals(q3.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
assertEquals(q4.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
}
public void testMaxCapacity() throws IOException {
this.setUp(4, 1, 1);
taskTrackerManager.addJobInProgressListener(scheduler.jobQueuesManager);
AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
QueueSchedulingContext a1 = new QueueSchedulingContext(
"R.a", 25, 50, -1);
QueueSchedulingContext a2 = new QueueSchedulingContext(
"R.b", 25, 30, -1);
QueueSchedulingContext a3 = new QueueSchedulingContext(
"R.c", 50, -1, -1);
//Test for max capacity
AbstractQueue q = new JobQueue(rt, a1);
AbstractQueue q1 = new JobQueue(rt, a2);
AbstractQueue q2 = new JobQueue(rt, a3);
scheduler.jobQueuesManager.addQueue((JobQueue) q);
scheduler.jobQueuesManager.addQueue((JobQueue) q1);
scheduler.jobQueuesManager.addQueue((JobQueue) q2);
scheduler.setRoot(rt);
rt.update(4, 4);
scheduler.updateContextInfoForTests();
// submit a job to the second queue
taskTrackerManager.submitJobAndInit(JobStatus.PREP, 20, 20, "R.a", "u1");
//Queue R.a should not more than 2 slots
Map<String, String> expectedStrings = new HashMap<String, String>();
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP,
"attempt_test_0001_m_000001_0 on tt1");
expectedStrings.put(
CapacityTestUtils.REDUCE,
"attempt_test_0001_r_000001_0 on tt1");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt1",
expectedStrings);
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP,
"attempt_test_0001_m_000002_0 on tt2");
expectedStrings.put(
CapacityTestUtils.REDUCE,
"attempt_test_0001_r_000002_0 on tt2");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt2",
expectedStrings);
assertNull(scheduler.assignTasks(
taskTrackerManager.getTaskTracker(
"tt3")));
}
public void testDistributedUnConfiguredCapacity() {
AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
//generate Queuecontext for the children
QueueSchedulingContext a1 = new QueueSchedulingContext(
"a", 50, -1, -1);
QueueSchedulingContext a2 = new QueueSchedulingContext(
"b", -1, -1, -1);
AbstractQueue rtChild1 = new ContainerQueue(rt, a1);
AbstractQueue rtChild2 = new ContainerQueue(rt, a2);
//Add further children to rtChild1.
QueueSchedulingContext b = new QueueSchedulingContext(
"ab", 30, -1, -1);
QueueSchedulingContext c = new QueueSchedulingContext(
"ac", -1, -1, -1);
QueueSchedulingContext d = new QueueSchedulingContext(
"ad", 100, -1, -1);
AbstractQueue q0 = new JobQueue(rtChild1, b);
AbstractQueue q1 = new JobQueue(rtChild1, c);
AbstractQueue q2 = new JobQueue(rtChild2, d);
rt.distributeUnConfiguredCapacity();
//after distribution the rtChild2 capacity should be 50.
assertEquals(
rtChild2.getQueueSchedulingContext().getCapacityPercent(), 50.0f);
//Now check the capacity of q1. It should be 60%.
assertTrue(q1.getQueueSchedulingContext().getCapacityPercent() == 70);
}
private Map setUpHierarchy() {
return setUpHierarchy(60, 40, 80, 20);
}
/**
* @param a capacity for sch q
* @param b capacity for gta q
* @param c capacity for sch.prod q
* @param d capacity for sch.misc q
* @return
*/
private Map<String, AbstractQueue> setUpHierarchy(
int a, int b, int c, int d) {
HashMap<String, AbstractQueue> map = new HashMap<String, AbstractQueue>();
AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
map.put(rt.getName(), rt);
scheduler.setRoot(rt);
// Create 2 levels of hierarchy.
//Firt level
QueueSchedulingContext sch =
new QueueSchedulingContext("rt.sch", a, -1, -1);
QueueSchedulingContext gta =
new QueueSchedulingContext("rt.gta", b, -1, -1);
AbstractQueue schq = new ContainerQueue(rt, sch);
//Cannot declare a ContainerQueue if no children.
AbstractQueue gtaq = new JobQueue(rt, gta);
map.put(schq.getName(), schq);
map.put(gtaq.getName(), gtaq);
scheduler.jobQueuesManager.addQueue((JobQueue) gtaq);
//Create further children.
QueueSchedulingContext prod =
new QueueSchedulingContext("rt.sch.prod", c, -1, -1);
QueueSchedulingContext misc =
new QueueSchedulingContext("rt.sch.misc", d, -1, -1);
AbstractQueue prodq = new JobQueue(schq, prod);
AbstractQueue miscq = new JobQueue(schq, misc);
map.put(prodq.getName(), prodq);
map.put(miscq.getName(), miscq);
scheduler.jobQueuesManager.addQueue(
(JobQueue) prodq);
scheduler.jobQueuesManager.addQueue((JobQueue) miscq);
return map;
}
/**
* Verifies that capacities are allocated properly in hierarchical queues.
*
* The test case sets up a hierarchy of queues and submits jobs to
* all the queues. It verifies that computation of capacities, sorting,
* etc are working correctly.
* @throws Exception
*/
public void testCapacityAllocationInHierarchicalQueues() throws Exception {
this.setUp(9, 1, 1);
taskTrackerManager.addJobInProgressListener(scheduler.jobQueuesManager);
// set up some queues
Map<String, AbstractQueue> map = setUpHierarchy();
scheduler.updateContextInfoForTests();
// verify initial capacity distribution
TaskSchedulingContext mapTsc
= map.get("rt.gta").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTsc.getCapacity(), 3);
mapTsc = map.get("rt.sch").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTsc.getCapacity(), 5);
mapTsc = map.get("rt.sch.prod").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTsc.getCapacity(), 4);
mapTsc = map.get("rt.sch.misc").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTsc.getCapacity(), 1);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{0, 0, 0, 0});
//Only Allow job submission to leaf queue
taskTrackerManager.submitJobAndInit(
JobStatus.PREP, 4, 4, "rt.sch.prod",
"u1");
// submit a job to the second queue
taskTrackerManager.submitJobAndInit(
JobStatus.PREP, 4, 4, "rt.sch.misc",
"u1");
//submit a job in gta level queue
taskTrackerManager.submitJobAndInit(JobStatus.PREP, 4, 4, "rt.gta", "u1");
int counter = 0;
Map<String, String> expectedStrings = new HashMap<String, String>();
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt1",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{0, 1, 1, 0});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0003_m_000001_0 on tt2");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0003_r_000001_0 on tt2");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt2",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{1, 1, 1, 0});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt3");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt3");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt3",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{1, 2, 1, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0003_m_000002_0 on tt4");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0003_r_000002_0 on tt4");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt4",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{2, 2, 1, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0001_m_000002_0 on tt5");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0001_r_000002_0 on tt5");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt5",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{2, 3, 2, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0001_m_000003_0 on tt6");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0001_r_000003_0 on tt6");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt6",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{2, 4, 3, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0003_m_000003_0 on tt7");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0003_r_000003_0 on tt7");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt7",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{3, 4, 3, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0001_m_000004_0 on tt8");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0001_r_000004_0 on tt8");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt8",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{3, 5, 4, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(
CapacityTestUtils.MAP, "attempt_test_0002_m_000002_0 on tt9");
expectedStrings.put(
CapacityTestUtils.REDUCE, "attempt_test_0002_r_000002_0 on tt9");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt9",
expectedStrings);
}
/**
* Test to make sure that capacity is divided at each level properly.
*
* The test case sets up a two level hierarchy of queues as follows:
* - rt.sch
* - rt.sch.prod
* - rt.sch.misc
* - rt.gta
* Jobs are submitted to rt.sch.misc and rt.gta, and the test verifies
* that as long as rt.sch is below rt.gta's capacity, it still gets
* allocated slots even if rt.sch.misc is over its capacity.
* @throws IOException
*/
public void testHierarchicalCapacityAllocation() throws IOException {
this.setUp(8, 1, 1);
taskTrackerManager.addJobInProgressListener(scheduler.jobQueuesManager);
// set up some queues
Map<String, AbstractQueue> map = setUpHierarchy(70, 30, 80, 20);
scheduler.updateContextInfoForTests();
// verify capacities as per the setup.
TaskSchedulingContext mapTSC
= map.get("rt.gta").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTSC.getCapacity(), 2);
mapTSC = map.get("rt.sch").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTSC.getCapacity(), 5);
mapTSC = map.get("rt.sch.prod").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTSC.getCapacity(), 4);
mapTSC = map.get("rt.sch.misc").getQueueSchedulingContext().getMapTSC();
assertEquals(mapTSC.getCapacity(), 1);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{0, 0, 0, 0});
// submit a job to the second queue
taskTrackerManager.submitJobAndInit(
JobStatus.PREP, 20, 20, "rt.sch.misc",
"u1");
//submit a job in gta level queue
taskTrackerManager.submitJobAndInit(JobStatus.PREP, 20, 20, "rt.gta", "u1");
int counter = 0;
Map<String, String> expectedStrings = new HashMap<String, String>();
expectedStrings.put(MAP, "attempt_test_0001_m_000001_0 on tt1");
expectedStrings.put(REDUCE, "attempt_test_0001_r_000001_0 on tt1");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt1",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{0, 1, 0, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(MAP, "attempt_test_0002_m_000001_0 on tt2");
expectedStrings.put(REDUCE, "attempt_test_0002_r_000001_0 on tt2");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt2",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{1, 1, 0, 1});
//============================================
expectedStrings.clear();
expectedStrings.put(MAP, "attempt_test_0001_m_000002_0 on tt3");
expectedStrings.put(REDUCE, "attempt_test_0001_r_000002_0 on tt3");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt3",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{1, 2, 0, 2});
//============================================
expectedStrings.clear();
expectedStrings.put(MAP, "attempt_test_0001_m_000003_0 on tt4");
expectedStrings.put(REDUCE, "attempt_test_0001_r_000003_0 on tt4");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt4",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{1, 3, 0, 3});
//============================================
expectedStrings.clear();
expectedStrings.put(MAP, "attempt_test_0002_m_000002_0 on tt5");
expectedStrings.put(REDUCE, "attempt_test_0002_r_000002_0 on tt5");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt5",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{2, 3, 0, 3});
//============================================
expectedStrings.clear();
expectedStrings.put(MAP, "attempt_test_0001_m_000004_0 on tt6");
expectedStrings.put(REDUCE, "attempt_test_0001_r_000004_0 on tt6");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt6",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{2, 4, 0, 4});
//============================================
expectedStrings.clear();
expectedStrings.put(MAP, "attempt_test_0001_m_000005_0 on tt7");
expectedStrings.put(REDUCE, "attempt_test_0001_r_000005_0 on tt7");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt7",
expectedStrings);
assertUsedCapacity(
map,
new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
new int[]{2, 5, 0, 5});
//============================================
expectedStrings.clear();
expectedStrings.put(MAP, "attempt_test_0001_m_000006_0 on tt8");
expectedStrings.put(REDUCE, "attempt_test_0001_r_000006_0 on tt8");
checkMultipleTaskAssignment(
taskTrackerManager, scheduler, "tt8",
expectedStrings);
}
// verify that the number of slots used for each queue
// matches the expected value.
private void assertUsedCapacity(Map<String, AbstractQueue> queueMap,
String[] queueNames, int[] expectedUsedSlots) {
scheduler.updateContextInfoForTests();
assertEquals(queueNames.length, expectedUsedSlots.length);
for (int i=0; i<queueNames.length; i++) {
TaskSchedulingContext mapTSC =
queueMap.get(queueNames[i]).getQueueSchedulingContext().getMapTSC();
assertEquals(mapTSC.getNumSlotsOccupied(), expectedUsedSlots[i]);
}
}
}