/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.huahinframework.manager.rest.service;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.JobStatus.State;
import org.apache.wink.common.internal.utils.MediaTypeUtils;
import org.apache.wink.common.model.multipart.InMultiPart;
import org.apache.wink.common.model.multipart.InPart;
import org.huahinframework.manager.queue.Queue;
import org.huahinframework.manager.queue.QueueUtils;
import org.huahinframework.manager.response.Response;
import org.huahinframework.manager.util.JobUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
/**
*
*/
@Path("/job")
public class JobService extends Service {
private static final Log log = LogFactory.getLog(JobService.class);
private static final String JOBID = "JobID";
private static final String JOBNAME = "JobName";
private static final String CONTENT_DISPOSITION = "Content-Disposition";
private static final String FORM_DATA_NAME_JAR = "form-data; name=\"JAR\"";
private static final String FORM_DATA_NAME_ARGUMENTS = "form-data; name=\"ARGUMENTS\"";
private static final String JSON_CLASS = "class";
private static final String JSON_SCRIPT = "script";
private static final String JSON_ARGUMENTS = "arguments";
private static final int JAR = 1;
private static final int ARGUMENTS = 2;
private static final Pattern fileNamePattern = Pattern.compile("^form-data; name=\"JAR\"; filename=\"(.*)\"$");
/**
* @return job {@link JSONArray}
* @throws JSONException
* @throws InterruptedException
*/
@Path("/list")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONArray list() throws JSONException, InterruptedException {
return JobUtils.getJobs(null, getJobConf());
}
/**
* @return job {@link JSONArray}
* @throws JSONException
* @throws InterruptedException
*/
@Path("/list/failed")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONArray listFailed() throws JSONException, InterruptedException {
return JobUtils.getJobs(State.FAILED, getJobConf());
}
/**
* @return job {@link JSONArray}
* @throws JSONException
* @throws InterruptedException
*/
@Path("/list/killed")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONArray listKilled() throws JSONException, InterruptedException {
return JobUtils.getJobs(State.KILLED, getJobConf());
}
/**
* @return job {@link JSONArray}
* @throws JSONException
* @throws InterruptedException
*/
@Path("/list/prep")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONArray listPrep() throws JSONException, InterruptedException {
return JobUtils.getJobs(State.PREP, getJobConf());
}
/**
* @return job {@link JSONArray}
* @throws JSONException
* @throws InterruptedException
*/
@Path("/list/running")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONArray listRunning() throws JSONException, InterruptedException {
return JobUtils.getJobs(State.RUNNING, getJobConf());
}
/**
* @return job {@link JSONArray}
* @throws JSONException
* @throws InterruptedException
*/
@Path("/list/succeeded")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONArray listSucceeded() throws JSONException, InterruptedException {
return JobUtils.getJobs(State.SUCCEEDED, getJobConf());
}
/**
* @return {@link JSONObject}
* @throws JSONException
* @throws InterruptedException
*/
@Path("/status/{" + JOBID + "}")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONObject status(@PathParam(JOBID) String jobId)
throws JSONException, InterruptedException {
JSONObject jsonObject = null;
try {
Map<String, Object> job = getStatus(jobId);
if (job != null) {
jsonObject = new JSONObject(job);
}
} catch (IOException e) {
e.printStackTrace();
log.error(e);
Map<String, String> status = new HashMap<String, String>();
status.put(Response.STATUS, e.getMessage());
jsonObject = new JSONObject(status);
}
if (jsonObject == null) {
Map<String, String> status = new HashMap<String, String>();
status.put(Response.STATUS, "Could not find job " + jobId);
jsonObject = new JSONObject(status);
}
return jsonObject;
}
/**
* @param jobId
* @return {@link JSONObject}
* @throws JSONException
*/
@Path("/detail/{" + JOBID + "}")
@GET
@Produces(MediaType.APPLICATION_JSON)
public JSONObject detail(@PathParam(JOBID) String jobId) throws JSONException {
JSONObject jsonObject = null;
JobConf conf = getJobConf();
try {
final Map<String, Object> job = getStatus(jobId);
if (job != null) {
Cluster cluster = new Cluster(conf);
Job j = cluster.getJob(JobID.forName(jobId));
if (j != null) {
String jobFile = j.getJobFile();
job.put(Response.JOB_FILE, jobFile);
job.put(Response.TRACKING_URL, j.getTrackingURL());
Map<String, String> jobConf = JobUtils.getJobConfiguration(jobFile, conf);
if (jobConf != null) {
job.put(Response.CONFIGURATION, jobConf);
}
}
jsonObject = new JSONObject(job);
}
} catch (Exception e) {
e.printStackTrace();
log.error(e);
Map<String, String> status = new HashMap<String, String>();
status.put(Response.STATUS, e.getMessage());
jsonObject = new JSONObject(status);
}
if (jsonObject == null) {
Map<String, String> status = new HashMap<String, String>();
status.put(Response.STATUS, "Could not find job " + jobId);
jsonObject = new JSONObject(status);
}
return jsonObject;
}
/**
* @param inMP
* @return {@link JSONObject}
*/
@Path("/register")
@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaTypeUtils.MULTIPART_FORM_DATA)
public JSONObject registerJob(InMultiPart inMP) {
Map<String, String> status = new HashMap<String, String>();
status.put(Response.STATUS, "accepted");
try {
Queue queue = new Queue();
synchronized (queue) {
queue.setDate(new Date());
}
boolean jarFound = false;
boolean argFound = false;
File jarFile = File.createTempFile("huahin", ".jar", new File(getJarPath()));
while (inMP.hasNext()) {
InPart part = inMP.next();
int type = 0;
for (String s : part.getHeaders().get(CONTENT_DISPOSITION)) {
if (s.startsWith(FORM_DATA_NAME_JAR)) {
Matcher matcher = fileNamePattern.matcher(s);
if (matcher.matches() && matcher.groupCount() == 1) {
queue.setJarFileName(matcher.group(1));
}
type = JAR;
jarFound = true;
break;
} else if (s.startsWith(FORM_DATA_NAME_ARGUMENTS)) {
type = ARGUMENTS;
argFound = true;
break;
}
}
InputStream in = part.getInputStream();
switch (type) {
case JAR:
createJar(in, jarFile);
queue.setJar(jarFile.getPath());
break;
case ARGUMENTS:
JSONObject argument = createJSON(in);
queue.setClazz(argument.getString(JSON_CLASS));
JSONArray array = argument.getJSONArray(JSON_ARGUMENTS);
String[] arguments = new String[array.length()];
for (int i = 0; i < array.length(); i++) {
arguments[i] = array.getString(i);
}
queue.setArguments(arguments);
break;
}
}
if (!jarFound || !argFound) {
jarFile.delete();
status.put(Response.STATUS, "arguments error");
return new JSONObject(status);
}
QueueUtils.registerQueue(getQueuePath(), queue);
} catch (Exception e) {
e.printStackTrace();
log.error(e);
status.put(Response.STATUS, e.getMessage());
}
return new JSONObject(status);
}
/**
* @param inMP
* @return {@link JSONObject}
*/
@Path("/register/hive")
@POST
@Produces(MediaType.APPLICATION_JSON)
public JSONObject registerHive(InMultiPart inMP) {
return registerScripts(Queue.TYPE_HIVE, inMP);
}
/**
* @param inMP
* @return {@link JSONObject}
*/
@Path("/register/pig")
@POST
@Produces(MediaType.APPLICATION_JSON)
public JSONObject registerPig(InMultiPart inMP) {
return registerScripts(Queue.TYPE_PIG, inMP);
}
/**
* @param type
* @param inMP
* @return JSONObject
*/
public JSONObject registerScripts(int type, InMultiPart inMP) {
Map<String, String> status = new HashMap<String, String>();
status.put(Response.STATUS, "accepted");
try {
Queue queue = new Queue();
queue.setType(type);
synchronized (queue) {
queue.setDate(new Date());
}
if (!inMP.hasNext()) {
status.put(Response.STATUS, "arguments error");
return new JSONObject(status);
}
InPart part = inMP.next();
InputStream in = part.getInputStream();
JSONObject argument = createJSON(in);
queue.setScript(argument.getString(JSON_SCRIPT));
QueueUtils.registerQueue(getQueuePath(), queue);
} catch (Exception e) {
e.printStackTrace();
log.error(e);
status.put(Response.STATUS, e.getMessage());
}
return new JSONObject(status);
}
/**
* @param in
* @param jarFile
* @throws IOException
*/
private void createJar(InputStream in, File jarFile) throws IOException {
OutputStream out = new FileOutputStream(jarFile);
byte[] buf = new byte[1024];
int len = 0;
while ((len = in.read(buf)) > 0) {
out.write(buf, 0, len);
}
out.close();
}
/**
* @return {@link JSONObject}
*/
@Path("/kill/id/{" + JOBID + "}")
@DELETE
@Produces(MediaType.APPLICATION_JSON)
public JSONObject killJobId(@PathParam(JOBID) String jobId) {
Map<String, String> status = new HashMap<String, String>();
try {
Cluster cluster = new Cluster(getJobConf());
for (JobStatus jobStatus : cluster.getAllJobStatuses()) {
if (jobStatus.getJobID().toString().equals(jobId)) {
Job job = cluster.getJob(jobStatus.getJobID());
if (job == null) {
break;
}
job.killJob();
status.put(Response.STATUS, "Killed job " + jobId);
break;
}
}
} catch (Exception e) {
e.printStackTrace();
log.error(e);
status.put(Response.STATUS, e.getMessage());
}
if (status.isEmpty()) {
status.put(Response.STATUS, "Could not find job " + jobId);
}
return new JSONObject(status);
}
/**
* @return {@link JSONObject}
*/
@Path("/kill/name/{" + JOBNAME + "}")
@DELETE
@Produces(MediaType.APPLICATION_JSON)
public JSONObject killJobName(@PathParam(JOBNAME) String jobName) {
Map<String, String> status = new HashMap<String, String>();
try {
Cluster cluster = new Cluster(getJobConf());
for (JobStatus jobStatus : cluster.getAllJobStatuses()) {
Job job = cluster.getJob(jobStatus.getJobID());
if (job == null) {
break;
}
if (job.getJobName().equals(jobName)) {
job.killJob();
status.put(Response.STATUS, "Killed job " + jobName);
break;
}
}
} catch (Exception e) {
e.printStackTrace();
log.error(e);
status.put(Response.STATUS, e.getMessage());
}
if (status.isEmpty()) {
status.put(Response.STATUS, "Could not find job " + jobName);
}
return new JSONObject(status);
}
/**
* @param jobId
* @return {@link JSONObject}
* @throws IOException
* @throws InterruptedException
*/
private Map<String, Object> getStatus(String jobId)
throws IOException, InterruptedException {
Map<String, Object> job = null;
Cluster cluster = new Cluster(getJobConf());
for (JobStatus jobStatus : cluster.getAllJobStatuses()) {
if (jobStatus.getJobID().toString().equals(jobId)) {
job = JobUtils.getJob(jobStatus);
Job j = cluster.getJob(jobStatus.getJobID());
if (j == null) {
break;
}
Calendar finishTime = Calendar.getInstance();
finishTime.setTimeInMillis(j.getFinishTime());
job.put(Response.FINISH_TIME, finishTime.getTime().toString());
Map<String, Map<String, Long>> groups = new HashMap<String, Map<String,Long>>();
for (String s : j.getCounters().getGroupNames()) {
CounterGroup counterGroup = j.getCounters().getGroup(s);
Iterator<Counter> ite = counterGroup.iterator();
Map<String, Long> counters = new HashMap<String, Long>();
groups.put(counterGroup.getDisplayName(), counters);
while (ite.hasNext()) {
Counter counter = (Counter) ite.next();
counters.put(counter.getDisplayName(), counter.getValue());
}
}
job.put(Response.GROUPS, groups);
break;
}
}
return job;
}
}