TestJobConf.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import java.util.regex.Pattern;
import static org.junit.Assert.*;

import org.apache.hadoop.fs.Path;
import org.junit.Assert;
import org.junit.Test;

/**
 * test JobConf
 * 
 */
public class TestJobConf {

  /**
   * test getters and setters of JobConf
   */
  @SuppressWarnings("deprecation")
  @Test (timeout=5000)
  public void testJobConf() {
    JobConf conf = new JobConf();
    // test default value
    Pattern pattern = conf.getJarUnpackPattern();
    assertEquals(Pattern.compile("(?:classes/|lib/).*").toString(),
        pattern.toString());
    // default value
    assertFalse(conf.getKeepFailedTaskFiles());
    conf.setKeepFailedTaskFiles(true);
    assertTrue(conf.getKeepFailedTaskFiles());

    // default value
    assertNull(conf.getKeepTaskFilesPattern());
    conf.setKeepTaskFilesPattern("123454");
    assertEquals("123454", conf.getKeepTaskFilesPattern());

    // default value
    assertNotNull(conf.getWorkingDirectory());
    conf.setWorkingDirectory(new Path("test"));
    assertTrue(conf.getWorkingDirectory().toString().endsWith("test"));

    // default value
    assertEquals(1, conf.getNumTasksToExecutePerJvm());

    // default value
    assertNull(conf.getKeyFieldComparatorOption());
    conf.setKeyFieldComparatorOptions("keySpec");
    assertEquals("keySpec", conf.getKeyFieldComparatorOption());

    // default value
    assertFalse(conf.getUseNewReducer());
    conf.setUseNewReducer(true);
    assertTrue(conf.getUseNewReducer());

    // default
    assertTrue(conf.getMapSpeculativeExecution());
    assertTrue(conf.getReduceSpeculativeExecution());
    assertTrue(conf.getSpeculativeExecution());
    conf.setReduceSpeculativeExecution(false);
    assertTrue(conf.getSpeculativeExecution());

    conf.setMapSpeculativeExecution(false);
    assertFalse(conf.getSpeculativeExecution());
    assertFalse(conf.getMapSpeculativeExecution());
    assertFalse(conf.getReduceSpeculativeExecution());

    conf.setSessionId("ses");
    assertEquals("ses", conf.getSessionId());

    assertEquals(3, conf.getMaxTaskFailuresPerTracker());
    conf.setMaxTaskFailuresPerTracker(2);
    assertEquals(2, conf.getMaxTaskFailuresPerTracker());

    assertEquals(0, conf.getMaxMapTaskFailuresPercent());
    conf.setMaxMapTaskFailuresPercent(50);
    assertEquals(50, conf.getMaxMapTaskFailuresPercent());

    assertEquals(0, conf.getMaxReduceTaskFailuresPercent());
    conf.setMaxReduceTaskFailuresPercent(70);
    assertEquals(70, conf.getMaxReduceTaskFailuresPercent());

    // by default
    assertEquals(JobPriority.NORMAL.name(), conf.getJobPriority().name());
    conf.setJobPriority(JobPriority.HIGH);
    assertEquals(JobPriority.HIGH.name(), conf.getJobPriority().name());

    assertNull(conf.getJobSubmitHostName());
    conf.setJobSubmitHostName("hostname");
    assertEquals("hostname", conf.getJobSubmitHostName());

    // default
    assertNull(conf.getJobSubmitHostAddress());
    conf.setJobSubmitHostAddress("ww");
    assertEquals("ww", conf.getJobSubmitHostAddress());

    // default value
    assertFalse(conf.getProfileEnabled());
    conf.setProfileEnabled(true);
    assertTrue(conf.getProfileEnabled());

    // default value
    assertEquals(conf.getProfileTaskRange(true).toString(), "0-2");
    assertEquals(conf.getProfileTaskRange(false).toString(), "0-2");
    conf.setProfileTaskRange(true, "0-3");
    assertEquals(conf.getProfileTaskRange(false).toString(), "0-2");
    assertEquals(conf.getProfileTaskRange(true).toString(), "0-3");

    // default value
    assertNull(conf.getMapDebugScript());
    conf.setMapDebugScript("mDbgScript");
    assertEquals("mDbgScript", conf.getMapDebugScript());

    // default value
    assertNull(conf.getReduceDebugScript());
    conf.setReduceDebugScript("rDbgScript");
    assertEquals("rDbgScript", conf.getReduceDebugScript());

    // default value
    assertNull(conf.getJobLocalDir());

    assertEquals("default", conf.getQueueName());
    conf.setQueueName("qname");
    assertEquals("qname", conf.getQueueName());

    assertEquals(1, conf.computeNumSlotsPerMap(100L));
    assertEquals(1, conf.computeNumSlotsPerReduce(100L));

    conf.setMemoryForMapTask(100 * 1000);
    assertEquals(1000, conf.computeNumSlotsPerMap(100L));
    conf.setMemoryForReduceTask(1000 * 1000);
    assertEquals(1000, conf.computeNumSlotsPerReduce(1000L));

    assertEquals(-1, conf.getMaxPhysicalMemoryForTask());
    assertEquals("The variable key is no longer used.",
        JobConf.deprecatedString("key"));

  }

  /**
   * Ensure that M/R 1.x applications can get and set task virtual memory with
   * old property names
   */
  @SuppressWarnings("deprecation")
  @Test (timeout = 1000)
  public void testDeprecatedPropertyNameForTaskVmem() {
    JobConf configuration = new JobConf();

    configuration.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024);
    configuration.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024);
    Assert.assertEquals(1024, configuration.getMemoryForMapTask());
    Assert.assertEquals(1024, configuration.getMemoryForReduceTask());
    // Make sure new property names aren't broken by the old ones
    configuration.setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, 1025);
    configuration.setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, 1025);
    Assert.assertEquals(1025, configuration.getMemoryForMapTask());
    Assert.assertEquals(1025, configuration.getMemoryForReduceTask());

    configuration.setMemoryForMapTask(2048);
    configuration.setMemoryForReduceTask(2048);
    Assert.assertEquals(2048, configuration.getLong(
        JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, -1));
    Assert.assertEquals(2048, configuration.getLong(
        JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
    // Make sure new property names aren't broken by the old ones
    Assert.assertEquals(2048, configuration.getLong(
        JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, -1));
    Assert.assertEquals(2048, configuration.getLong(
        JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
  }
}