MockHistoryJobs.java example

Explorer
HDP-2.2-Patched-master
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.v2.hs;

import java.io.IOException;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.ApplicationId;

import com.google.common.collect.Maps;

public class MockHistoryJobs extends MockJobs {

  public static class JobsPair {
    public Map<JobId, Job> partial;
    public Map<JobId, Job> full;
  }
  
  public static JobsPair newHistoryJobs(int numJobs, int numTasksPerJob,
      int numAttemptsPerTask) throws IOException {
    Map<JobId, Job> mocked = newJobs(numJobs, numTasksPerJob, numAttemptsPerTask);
    return split(mocked);
  }
  
  public static JobsPair newHistoryJobs(ApplicationId appID, int numJobsPerApp,
      int numTasksPerJob, int numAttemptsPerTask) throws IOException {
    Map<JobId, Job> mocked = newJobs(appID, numJobsPerApp, numTasksPerJob, 
        numAttemptsPerTask);
    return split(mocked);
  }
  
  public static JobsPair newHistoryJobs(ApplicationId appID, int numJobsPerApp,
      int numTasksPerJob, int numAttemptsPerTask, boolean hasFailedTasks)
      throws IOException {
    Map<JobId, Job> mocked = newJobs(appID, numJobsPerApp, numTasksPerJob,
        numAttemptsPerTask, hasFailedTasks);
    return split(mocked);
  }

  private static JobsPair split(Map<JobId, Job> mocked) throws IOException {
    JobsPair ret = new JobsPair();
    ret.full = Maps.newHashMap();
    ret.partial = Maps.newHashMap();
    for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
      JobId id = entry.getKey();
      Job j = entry.getValue();
      MockCompletedJob mockJob = new MockCompletedJob(j);
      // use MockCompletedJob to set everything below to make sure
      // consistent with what history server would do
      ret.full.put(id, mockJob);
      JobReport report = mockJob.getReport();
      JobIndexInfo info = new JobIndexInfo(report.getStartTime(), 
          report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id, 
          mockJob.getCompletedMaps(), mockJob.getCompletedReduces(),
          String.valueOf(mockJob.getState()));
      info.setQueueName(mockJob.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));

    }
    return ret;
  }

  private static class MockCompletedJob extends CompletedJob {
    private Job job;
    
    public MockCompletedJob(Job job) throws IOException {
      super(new Configuration(), job.getID(), null, true, job.getUserName(),
          null, null);
      this.job = job;
    }

    @Override
    public int getCompletedMaps() {
      // we always return total since this is history server
      // and PartialJob also assumes completed - total
      return job.getTotalMaps();
    }

    @Override
    public int getCompletedReduces() {
      // we always return total since this is history server
      // and PartialJob also assumes completed - total
      return job.getTotalReduces();
    }

    @Override
    public org.apache.hadoop.mapreduce.Counters getAllCounters() {
      return job.getAllCounters();
    }

    @Override
    public JobId getID() {
      return job.getID();
    }

    @Override
    public JobReport getReport() {
      return job.getReport();
    }

    @Override
    public float getProgress() {
      return job.getProgress();
    }

    @Override
    public JobState getState() {
      return job.getState();
    }

    @Override
    public Task getTask(TaskId taskId) {
      return job.getTask(taskId);
    }

    @Override
    public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
        int fromEventId, int maxEvents) {
      return job.getTaskAttemptCompletionEvents(fromEventId, maxEvents);
    }

    @Override
    public TaskCompletionEvent[] getMapAttemptCompletionEvents(
        int startIndex, int maxEvents) {
      return job.getMapAttemptCompletionEvents(startIndex, maxEvents);
    }

    @Override
    public Map<TaskId, Task> getTasks() {
      return job.getTasks();
    }

    @Override
    protected void loadFullHistoryData(boolean loadTasks,
        Path historyFileAbsolute) throws IOException {
      //Empty
    }

    @Override
    public List<String> getDiagnostics() {
      return job.getDiagnostics();
    }

    @Override
    public String getName() {
      return job.getName();
    }

    @Override
    public String getQueueName() {
      return job.getQueueName();
    }

    @Override
    public int getTotalMaps() {
      return job.getTotalMaps();
    }

    @Override
    public int getTotalReduces() {
      return job.getTotalReduces();
    }

    @Override
    public boolean isUber() {
      return job.isUber();
    }

    @Override
    public Map<TaskId, Task> getTasks(TaskType taskType) {
      return job.getTasks();
    }

    @Override
    public
        boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation) {
      return job.checkAccess(callerUGI, jobOperation);
    }
    
    @Override
    public  Map<JobACL, AccessControlList> getJobACLs() {
      return job.getJobACLs();
    }
    
    @Override
    public String getUserName() {
      return job.getUserName();
    }

    @Override
    public Path getConfFile() {
      return job.getConfFile();
    }

    @Override
    public List<AMInfo> getAMInfos() {
      return job.getAMInfos();
    }
  }
}