/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.YarnException;
import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
public class WebHCatJTShim23 implements WebHCatJTShim {
private static final Logger LOG = LoggerFactory.getLogger(WebHCatJTShim23.class);
private JobClient jc;
private final Configuration conf;
/**
* Create a connection to the Job Tracker.
*/
public WebHCatJTShim23(final Configuration conf, final UserGroupInformation ugi)
throws IOException {
try {
this.conf = conf;
jc = ugi.doAs(new PrivilegedExceptionAction<JobClient>() {
public JobClient run() throws IOException, InterruptedException {
//create this in doAs() so that it gets a security context based passed in 'ugi'
return new JobClient(conf);
}
});
}
catch(InterruptedException ex) {
throw new RuntimeException("Failed to create JobClient", ex);
}
}
/**
* Grab a handle to a job that is already known to the JobTracker.
*
* @return Profile of the job, or null if not found.
*/
public JobProfile getJobProfile(JobID jobid)
throws IOException {
RunningJob rj = getJob(jobid);
if(rj == null) {
return null;
}
JobStatus jobStatus = rj.getJobStatus();
return new JobProfile(jobStatus.getUsername(), jobStatus.getJobID(),
jobStatus.getJobFile(), jobStatus.getTrackingUrl(), jobStatus.getJobName());
}
/**
* Grab a handle to a job that is already known to the JobTracker.
*
* @return Status of the job, or null if not found.
*/
public JobStatus getJobStatus(JobID jobid)
throws IOException {
RunningJob rj = getJob(jobid);
if(rj == null) {
return null;
}
return rj.getJobStatus();
}
/**
* Kill a job.
*/
public void killJob(JobID jobid)
throws IOException {
RunningJob rj = getJob(jobid);
if(rj == null) {
return;
}
rj.killJob();
}
/**
* Get all the jobs submitted.
*/
public JobStatus[] getAllJobs()
throws IOException {
return jc.getAllJobs();
}
/**
* Close the connection to the Job Tracker.
*/
public void close() {
try {
jc.close();
} catch (IOException e) {
}
}
@Override
public void addCacheFile(URI uri, Job job) {
job.addCacheFile(uri);
}
/**
* @return {@code null} if no such application exists
*/
private RunningJob getJob(JobID jobid) throws IOException {
try {
return jc.getJob(jobid);
}
catch(IOException ex) {
String msg = ex.getMessage();
if(msg != null && (msg.contains("ApplicationNotFoundException") ||
Pattern.compile("History file.*not found").matcher(msg).find())) {
LOG.info("Job(" + jobid + ") not found: " + msg);
return null;
}
throw ex;
}
}
/**
* Kills all jobs tagged with the given tag that have been started after the
* given timestamp.
*/
@Override
public void killJobs(String tag, long timestamp) {
try {
LOG.info("Looking for jobs to kill...");
Set<ApplicationId> childJobs = getYarnChildJobs(tag, timestamp);
if (childJobs.isEmpty()) {
LOG.info("No jobs found from");
return;
} else {
LOG.info(String.format("Found MR jobs count: %d", childJobs.size()));
LOG.info("Killing all found jobs");
YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);
yarnClient.start();
for (ApplicationId app: childJobs) {
LOG.info(String.format("Killing job: %s ...", app));
yarnClient.killApplication(app);
LOG.info(String.format("Job %s killed", app));
}
}
} catch (YarnException ye) {
throw new RuntimeException("Exception occurred while killing child job(s)", ye);
} catch (IOException ioe) {
throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
}
}
/**
* Returns all jobs tagged with the given tag that have been started after the
* given timestamp. Returned jobIds are MapReduce JobIds.
*/
@Override
public Set<String> getJobs(String tag, long timestamp) {
Set<ApplicationId> childYarnJobs = getYarnChildJobs(tag, timestamp);
Set<String> childJobs = new HashSet<String>();
for(ApplicationId id : childYarnJobs) {
// Convert to a MapReduce job id
String childJobId = TypeConverter.fromYarn(id).toString();
childJobs.add(childJobId);
}
return childJobs;
}
/**
* Queries RM for the list of applications with the given tag that have started
* after the given timestamp.
*/
private Set<ApplicationId> getYarnChildJobs(String tag, long timestamp) {
Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
LOG.info(String.format("Querying RM for tag = %s, starting with ts = %s", tag, timestamp));
GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
gar.setScope(ApplicationsRequestScope.OWN);
gar.setStartRange(timestamp, System.currentTimeMillis());
gar.setApplicationTags(Collections.singleton(tag));
try {
ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf,
ApplicationClientProtocol.class);
GetApplicationsResponse apps = proxy.getApplications(gar);
List<ApplicationReport> appsList = apps.getApplicationList();
for(ApplicationReport appReport : appsList) {
childYarnJobs.add(appReport.getApplicationId());
}
} catch (IOException ioe) {
throw new RuntimeException("Exception occurred while finding child jobs", ioe);
} catch (YarnException ye) {
throw new RuntimeException("Exception occurred while finding child jobs", ye);
}
return childYarnJobs;
}
}