/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.tools.rumen;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.jobhistory.JhCounter;
import org.apache.hadoop.mapreduce.jobhistory.JhCounterGroup;
import org.apache.hadoop.mapreduce.jobhistory.JhCounters;
import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
/**
* Job History related utils for handling multiple formats of history logs of
* different hadoop versions like Pre21 history logs, current history logs.
*/
public class JobHistoryUtils {
private static String applyParser(String fileName, Pattern pattern) {
Matcher matcher = pattern.matcher(fileName);
if (!matcher.matches()) {
return null;
}
return matcher.group(1);
}
/**
* Extracts jobID string from the given job history log file name or
* job history configuration file name.
* @param fileName name of job history file or job history configuration file
* @return a valid jobID String, parsed out of the file name. Otherwise,
* [especially for .crc files] returns null.
*/
static String extractJobID(String fileName) {
// Get jobID if fileName is a config file name.
String jobId = extractJobIDFromConfFileName(fileName);
if (jobId == null) {
// Get JobID if fileName is a job history file name
jobId = extractJobIDFromHistoryFileName(fileName);
}
return jobId;
}
/**
* Extracts job id from the current hadoop version's job history file name.
* @param fileName job history file name from which job id is to be extracted
* @return job id if the history file name format is same as that of the
* current hadoop version. Returns null otherwise.
*/
private static String extractJobIDFromCurrentHistoryFile(String fileName) {
JobID id = null;
if (org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils
.isValidJobHistoryFileName(fileName)) {
try {
id = org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils
.getJobIDFromHistoryFilePath(fileName);
} catch (IOException e) {
// Ignore this exception and go ahead with getting of jobID assuming
// older hadoop verison's history file
}
}
if (id != null) {
return id.toString();
}
return null;
}
/**
* Extracts jobID string from the given job history file name.
* @param fileName name of the job history file
* @return JobID if the given <code>fileName</code> is a valid job history
* file name, <code>null</code> otherwise.
*/
private static String extractJobIDFromHistoryFileName(String fileName) {
// History file name could be in one of the following formats
// (1) old pre21 job history file name format
// (2) new pre21 job history file name format
// (3) current job history file name format i.e. 0.22
// Try to get the jobID assuming that the history file is from the current
// hadoop version
String jobID = extractJobIDFromCurrentHistoryFile(fileName);
if (jobID != null) {
return jobID;//history file is of current hadoop version
}
// History file could be of older hadoop versions
String pre21JobID = applyParser(fileName,
Pre21JobHistoryConstants.JOBHISTORY_FILENAME_REGEX_V1);
if (pre21JobID == null) {
pre21JobID = applyParser(fileName,
Pre21JobHistoryConstants.JOBHISTORY_FILENAME_REGEX_V2);
}
return pre21JobID;
}
/**
* Extracts jobID string from the given job conf xml file name.
* @param fileName name of the job conf xml file
* @return job id if the given <code>fileName</code> is a valid job conf xml
* file name, <code>null</code> otherwise.
*/
private static String extractJobIDFromConfFileName(String fileName) {
// History conf file name could be in one of the following formats
// (1) old pre21 job history file name format
// (2) new pre21 job history file name format
// (3) current job history file name format i.e. 0.22
String pre21JobID = applyParser(fileName,
Pre21JobHistoryConstants.CONF_FILENAME_REGEX_V1);
if (pre21JobID == null) {
pre21JobID = applyParser(fileName,
Pre21JobHistoryConstants.CONF_FILENAME_REGEX_V2);
}
if (pre21JobID != null) {
return pre21JobID;
}
return applyParser(fileName, JobHistory.CONF_FILENAME_REGEX);
}
/**
* Checks if the given <code>fileName</code> is a valid job conf xml file name
* @param fileName name of the file to be validated
* @return <code>true</code> if the given <code>fileName</code> is a valid
* job conf xml file name.
*/
static boolean isJobConfXml(String fileName) {
String jobId = extractJobIDFromConfFileName(fileName);
return jobId != null;
}
/**
* Extract/Add counters into the Map from the given JhCounters object.
* @param counters the counters to be extracted from
* @return the map of counters
*/
static Map<String, Long> extractCounters(JhCounters counters) {
Map<String, Long> countersMap = new HashMap<String, Long>();
if (counters != null) {
for (JhCounterGroup group : counters.groups) {
for (JhCounter counter : group.counts) {
countersMap.put(counter.name.toString(), counter.value);
}
}
}
return countersMap;
}
}