/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hive.hcatalog.templeton;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.DELETE;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import org.apache.commons.exec.ExecuteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.apache.hive.hcatalog.templeton.LauncherDelegator.JobType;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
/**
* The Templeton Web API server.
*/
@Path("/v1")
public class Server {
public static final String VERSION = "v1";
public static final String DO_AS_PARAM = "doAs";
/**
* The status message. Always "ok"
*/
public static final Map<String, String> STATUS_OK = createStatusMsg();
/**
* The list of supported api versions.
*/
public static final Map<String, Object> SUPPORTED_VERSIONS = createVersions();
/**
* The list of supported return formats. Always json.
*/
public static final Map<String, Object> SUPPORTED_FORMATS = createFormats();
// Build the status message for the /status call.
private static Map<String, String> createStatusMsg() {
HashMap<String, String> res = new HashMap<String, String>();
res.put("status", "ok");
res.put("version", VERSION);
return Collections.unmodifiableMap(res);
}
// Build the versions list.
private static Map<String, Object> createVersions() {
ArrayList<String> versions = new ArrayList<String>();
versions.add(VERSION);
HashMap<String, Object> res = new HashMap<String, Object>();
res.put("supportedVersions", versions);
res.put("version", VERSION);
return Collections.unmodifiableMap(res);
}
// Build the supported formats list
private static Map<String, Object> createFormats() {
ArrayList<String> formats = new ArrayList<String>();
formats.add(MediaType.APPLICATION_JSON);
HashMap<String, Object> res = new HashMap<String, Object>();
res.put("responseTypes", formats);
return Collections.unmodifiableMap(res);
}
protected static ExecService execService = ExecServiceImpl.getInstance();
private static AppConfig appConf = Main.getAppConfigInstance();
// The SecurityContext set by AuthFilter
private
@Context
SecurityContext theSecurityContext;
// The uri requested
private
@Context
UriInfo theUriInfo;
private @QueryParam(DO_AS_PARAM) String doAs;
private @Context HttpServletRequest request;
private static final Logger LOG = LoggerFactory.getLogger(Server.class);
/**
* Check the status of this server. Always OK.
*/
@GET
@Path("status")
@Produces({MediaType.APPLICATION_JSON})
public Map<String, String> status() {
return STATUS_OK;
}
/**
* Check the supported request formats of this server.
*/
@GET
@Produces({MediaType.APPLICATION_JSON})
public Map<String, Object> requestFormats() {
return SUPPORTED_FORMATS;
}
/**
* Check the version(s) supported by this server.
*/
@GET
@Path("version")
@Produces({MediaType.APPLICATION_JSON})
public Map<String, Object> version() {
return SUPPORTED_VERSIONS;
}
/**
* Get version of hadoop software being run by this WebHCat server
*/
@GET
@Path("version/hadoop")
@Produces(MediaType.APPLICATION_JSON)
public Response hadoopVersion() throws IOException {
VersionDelegator d = new VersionDelegator(appConf);
return d.getVersion("hadoop");
}
/**
* Get version of hive software being run by this WebHCat server
*/
@GET
@Path("version/hive")
@Produces(MediaType.APPLICATION_JSON)
public Response hiveVersion() throws IOException {
VersionDelegator d = new VersionDelegator(appConf);
return d.getVersion("hive");
}
/**
* Get version of sqoop software being run by this WebHCat server
*/
@GET
@Path("version/sqoop")
@Produces(MediaType.APPLICATION_JSON)
public Response sqoopVersion() throws IOException {
VersionDelegator d = new VersionDelegator(appConf);
return d.getVersion("sqoop");
}
/**
* Get version of pig software being run by this WebHCat server
*/
@GET
@Path("version/pig")
@Produces(MediaType.APPLICATION_JSON)
public Response pigVersion() throws IOException {
VersionDelegator d = new VersionDelegator(appConf);
return d.getVersion("pig");
}
/**
* Execute an hcat ddl expression on the local box. It is run
* as the authenticated user and rate limited.
*/
@POST
@Path("ddl")
@Produces({MediaType.APPLICATION_JSON})
public ExecBean ddl(@FormParam("exec") String exec,
@FormParam("group") String group,
@FormParam("permissions") String permissions)
throws NotAuthorizedException, BusyException, BadParam,
ExecuteException, IOException {
verifyUser();
verifyParam(exec, "exec");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.run(getDoAsUser(), exec, false, group, permissions);
}
/**
* List all the tables in an hcat database.
*/
@GET
@Path("ddl/database/{db}/table")
@Produces(MediaType.APPLICATION_JSON)
public Response listTables(@PathParam("db") String db,
@QueryParam("like") String tablePattern)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
HcatDelegator d = new HcatDelegator(appConf, execService);
if (!TempletonUtils.isset(tablePattern)) {
tablePattern = "*";
}
return d.listTables(getDoAsUser(), db, tablePattern);
}
/**
* Create a new table.
*/
@PUT
@Path("ddl/database/{db}/table/{table}")
@Produces(MediaType.APPLICATION_JSON)
public Response createTable(@PathParam("db") String db,
@PathParam("table") String table,
TableDesc desc)
throws SimpleWebException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
desc.table = table;
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.createTable(getDoAsUser(), db, desc);
}
/**
* Create a new table like another table.
*/
@PUT
@Path("ddl/database/{db}/table/{existingTable}/like/{newTable}")
@Produces(MediaType.APPLICATION_JSON)
public Response createTableLike(@PathParam("db") String db,
@PathParam("existingTable") String existingTable,
@PathParam("newTable") String newTable,
TableLikeDesc desc)
throws SimpleWebException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(existingTable, ":existingTable");
verifyDdlParam(newTable, ":newTable");
desc.existingTable = existingTable;
desc.newTable = newTable;
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.createTableLike(getDoAsUser(), db, desc);
}
/**
* Describe an hcat table. This is normally a simple list of
* columns (using "desc table"), but the extended format will show
* more information (using "show table extended like").
*/
@GET
@Path("ddl/database/{db}/table/{table}")
@Produces(MediaType.APPLICATION_JSON)
public Response descTable(@PathParam("db") String db,
@PathParam("table") String table,
@QueryParam("format") String format)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
if ("extended".equals(format)) {
return d.descExtendedTable(getDoAsUser(), db, table);
}
else {
return d.descTable(getDoAsUser(), db, table, false);
}
}
/**
* Drop an hcat table.
*/
@DELETE
@Path("ddl/database/{db}/table/{table}")
@Produces(MediaType.APPLICATION_JSON)
public Response dropTable(@PathParam("db") String db,
@PathParam("table") String table,
@QueryParam("ifExists") boolean ifExists,
@QueryParam("group") String group,
@QueryParam("permissions") String permissions)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.dropTable(getDoAsUser(), db, table, ifExists, group, permissions);
}
/**
* Rename an hcat table.
*/
@POST
@Path("ddl/database/{db}/table/{table}")
@Produces(MediaType.APPLICATION_JSON)
public Response renameTable(@PathParam("db") String db,
@PathParam("table") String oldTable,
@FormParam("rename") String newTable,
@FormParam("group") String group,
@FormParam("permissions") String permissions)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(oldTable, ":table");
verifyDdlParam(newTable, "rename");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.renameTable(getDoAsUser(), db, oldTable, newTable, group, permissions);
}
/**
* Describe a single property on an hcat table.
*/
@GET
@Path("ddl/database/{db}/table/{table}/property/{property}")
@Produces(MediaType.APPLICATION_JSON)
public Response descOneTableProperty(@PathParam("db") String db,
@PathParam("table") String table,
@PathParam("property") String property)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
verifyPropertyParam(property, ":property");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.descTableProperty(getDoAsUser(), db, table, property);
}
/**
* List all the properties on an hcat table.
*/
@GET
@Path("ddl/database/{db}/table/{table}/property")
@Produces(MediaType.APPLICATION_JSON)
public Response listTableProperties(@PathParam("db") String db,
@PathParam("table") String table)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.listTableProperties(getDoAsUser(), db, table);
}
/**
* Add a single property on an hcat table.
*/
@PUT
@Path("ddl/database/{db}/table/{table}/property/{property}")
@Produces(MediaType.APPLICATION_JSON)
public Response addOneTableProperty(@PathParam("db") String db,
@PathParam("table") String table,
@PathParam("property") String property,
TablePropertyDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
verifyPropertyParam(property, ":property");
desc.name = property;
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.addOneTableProperty(getDoAsUser(), db, table, desc);
}
/**
* List all the partitions in an hcat table.
*/
@GET
@Path("ddl/database/{db}/table/{table}/partition")
@Produces(MediaType.APPLICATION_JSON)
public Response listPartitions(@PathParam("db") String db,
@PathParam("table") String table)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.listPartitions(getDoAsUser(), db, table);
}
/**
* Describe a single partition in an hcat table.
*/
@GET
@Path("ddl/database/{db}/table/{table}/partition/{partition}")
@Produces(MediaType.APPLICATION_JSON)
public Response descPartition(@PathParam("db") String db,
@PathParam("table") String table,
@PathParam("partition") String partition)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
verifyParam(partition, ":partition");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.descOnePartition(getDoAsUser(), db, table, partition);
}
/**
* Create a partition in an hcat table.
*/
@PUT
@Path("ddl/database/{db}/table/{table}/partition/{partition}")
@Produces(MediaType.APPLICATION_JSON)
public Response addOnePartition(@PathParam("db") String db,
@PathParam("table") String table,
@PathParam("partition") String partition,
PartitionDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
verifyParam(partition, ":partition");
desc.partition = partition;
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.addOnePartition(getDoAsUser(), db, table, desc);
}
/**
* Drop a partition in an hcat table.
*/
@DELETE
@Path("ddl/database/{db}/table/{table}/partition/{partition}")
@Produces(MediaType.APPLICATION_JSON)
public Response dropPartition(@PathParam("db") String db,
@PathParam("table") String table,
@PathParam("partition") String partition,
@QueryParam("ifExists") boolean ifExists,
@QueryParam("group") String group,
@QueryParam("permissions") String permissions)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
verifyParam(partition, ":partition");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.dropPartition(getDoAsUser(), db, table, partition, ifExists,
group, permissions);
}
/**
* List all databases, or those that match a pattern.
*/
@GET
@Path("ddl/database/")
@Produces(MediaType.APPLICATION_JSON)
public Response listDatabases(@QueryParam("like") String dbPattern)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
HcatDelegator d = new HcatDelegator(appConf, execService);
if (!TempletonUtils.isset(dbPattern)) {
dbPattern = "*";
}
return d.listDatabases(getDoAsUser(), dbPattern);
}
/**
* Describe a database
*/
@GET
@Path("ddl/database/{db}")
@Produces(MediaType.APPLICATION_JSON)
public Response descDatabase(@PathParam("db") String db,
@QueryParam("format") String format)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.descDatabase(getDoAsUser(), db, "extended".equals(format));
}
/**
* Create a database
*/
@PUT
@Path("ddl/database/{db}")
@Produces(MediaType.APPLICATION_JSON)
public Response createDatabase(@PathParam("db") String db,
DatabaseDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
desc.database = db;
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.createDatabase(getDoAsUser(), desc);
}
/**
* Drop a database
*/
@DELETE
@Path("ddl/database/{db}")
@Produces(MediaType.APPLICATION_JSON)
public Response dropDatabase(@PathParam("db") String db,
@QueryParam("ifExists") boolean ifExists,
@QueryParam("option") String option,
@QueryParam("group") String group,
@QueryParam("permissions") String permissions)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
if (TempletonUtils.isset(option)) {
verifyDdlParam(option, "option");
}
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.dropDatabase(getDoAsUser(), db, ifExists, option,
group, permissions);
}
/**
* List the columns in an hcat table. Currently the same as
* describe table.
*/
@GET
@Path("ddl/database/{db}/table/{table}/column")
@Produces(MediaType.APPLICATION_JSON)
public Response listColumns(@PathParam("db") String db,
@PathParam("table") String table)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.listColumns(getDoAsUser(), db, table);
}
/**
* Describe a single column in an hcat table.
*/
@GET
@Path("ddl/database/{db}/table/{table}/column/{column}")
@Produces(MediaType.APPLICATION_JSON)
public Response descColumn(@PathParam("db") String db,
@PathParam("table") String table,
@PathParam("column") String column)
throws SimpleWebException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
verifyParam(column, ":column");
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.descOneColumn(getDoAsUser(), db, table, column);
}
/**
* Create a column in an hcat table.
*/
@PUT
@Path("ddl/database/{db}/table/{table}/column/{column}")
@Produces(MediaType.APPLICATION_JSON)
public Response addOneColumn(@PathParam("db") String db,
@PathParam("table") String table,
@PathParam("column") String column,
ColumnDesc desc)
throws HcatException, NotAuthorizedException, BusyException,
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
verifyDdlParam(table, ":table");
verifyParam(column, ":column");
verifyParam(desc.type, "type");
desc.name = column;
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.addOneColumn(getDoAsUser(), db, table, desc);
}
/**
* Run a MapReduce Streaming job.
* @param callback URL which WebHCat will call when the hive job finishes
*/
@POST
@Path("mapreduce/streaming")
@Produces({MediaType.APPLICATION_JSON})
public EnqueueBean mapReduceStreaming(@FormParam("input") List<String> inputs,
@FormParam("inputreader") String inputreader,
@FormParam("output") String output,
@FormParam("mapper") String mapper,
@FormParam("reducer") String reducer,
@FormParam("combiner") String combiner,
@FormParam("file") List<String> fileList,
@FormParam("files") String files,
@FormParam("define") List<String> defines,
@FormParam("cmdenv") List<String> cmdenvs,
@FormParam("arg") List<String> args,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
@FormParam("enablelog") boolean enablelog,
@FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException, TooManyRequestsException {
verifyUser();
verifyParam(inputs, "input");
verifyParam(mapper, "mapper");
verifyParam(reducer, "reducer");
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
userArgs.put("input", inputs);
userArgs.put("inputreader", inputreader);
userArgs.put("output", output);
userArgs.put("mapper", mapper);
userArgs.put("reducer", reducer);
userArgs.put("combiner", combiner);
userArgs.put("file", fileList);
userArgs.put("files", files);
userArgs.put("define", defines);
userArgs.put("cmdenv", cmdenvs);
userArgs.put("arg", args);
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
StreamingDelegator d = new StreamingDelegator(appConf);
return d.run(getDoAsUser(), userArgs, inputs, inputreader, output, mapper, reducer, combiner,
fileList, files, defines, cmdenvs, args,
statusdir, callback, getCompletedUrl(), enablelog, enablejobreconnect, JobType.STREAMING);
}
/**
* Run a MapReduce Jar job.
* Params correspond to the REST api params
* @param usesHcatalog if {@code true}, means the Jar uses HCat and thus needs to access
* metastore, which requires additional steps for WebHCat to perform in a secure cluster.
* @param callback URL which WebHCat will call when the hive job finishes
* @see org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob
*/
@POST
@Path("mapreduce/jar")
@Produces({MediaType.APPLICATION_JSON})
public EnqueueBean mapReduceJar(@FormParam("jar") String jar,
@FormParam("class") String mainClass,
@FormParam("libjars") String libjars,
@FormParam("files") String files,
@FormParam("arg") List<String> args,
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
@FormParam("usehcatalog") boolean usesHcatalog,
@FormParam("enablelog") boolean enablelog,
@FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException, TooManyRequestsException {
verifyUser();
verifyParam(jar, "jar");
verifyParam(mainClass, "class");
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
userArgs.put("jar", jar);
userArgs.put("class", mainClass);
userArgs.put("libjars", libjars);
userArgs.put("files", files);
userArgs.put("arg", args);
userArgs.put("define", defines);
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
JarDelegator d = new JarDelegator(appConf);
return d.run(getDoAsUser(), userArgs,
jar, mainClass,
libjars, files, args, defines,
statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog, enablejobreconnect, JobType.JAR);
}
/**
* Run a Pig job.
* Params correspond to the REST api params. If '-useHCatalog' is in the {@code pigArgs, usesHcatalog},
* is interpreted as true.
* @param usesHcatalog if {@code true}, means the Pig script uses HCat and thus needs to access
* metastore, which requires additional steps for WebHCat to perform in a secure cluster.
* This does nothing to ensure that Pig is installed on target node in the cluster.
* @param callback URL which WebHCat will call when the hive job finishes
* @see org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob
*/
@POST
@Path("pig")
@Produces({MediaType.APPLICATION_JSON})
public EnqueueBean pig(@FormParam("execute") String execute,
@FormParam("file") String srcFile,
@FormParam("arg") List<String> pigArgs,
@FormParam("files") String otherFiles,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
@FormParam("usehcatalog") boolean usesHcatalog,
@FormParam("enablelog") boolean enablelog,
@FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException, TooManyRequestsException {
verifyUser();
if (execute == null && srcFile == null) {
throw new BadParam("Either execute or file parameter required");
}
//add all function arguments to a map
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
userArgs.put("execute", execute);
userArgs.put("file", srcFile);
userArgs.put("arg", pigArgs);
userArgs.put("files", otherFiles);
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
PigDelegator d = new PigDelegator(appConf);
return d.run(getDoAsUser(), userArgs,
execute, srcFile,
pigArgs, otherFiles,
statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog, enablejobreconnect);
}
/**
* Run a Sqoop job.
* @param optionsFile name of option file which contains Sqoop command to run
* @param otherFiles additional files to be shipped to the launcher, such as option
files which contain part of the Sqoop command
* @param libdir dir containing JDBC jars that Sqoop will need to interact with the database
* @param statusdir where the stderr/stdout of templeton controller job goes
* @param callback URL which WebHCat will call when the sqoop job finishes
* @param enablelog whether to collect mapreduce log into statusdir/logs
* @param enablejobreconnect whether to reconnect to a running child job on templeton
* controller job retry
*/
@POST
@Path("sqoop")
@Produces({MediaType.APPLICATION_JSON})
public EnqueueBean sqoop(@FormParam("command") String command,
@FormParam("optionsfile") String optionsFile,
@FormParam("libdir") String libdir,
@FormParam("files") String otherFiles,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
@FormParam("enablelog") boolean enablelog,
@FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
IOException, InterruptedException, TooManyRequestsException {
verifyUser();
if (command == null && optionsFile == null)
throw new BadParam("Must define Sqoop command or a optionsfile contains Sqoop command to run Sqoop job.");
if (command != null && optionsFile != null)
throw new BadParam("Cannot set command and optionsfile at the same time.");
checkEnableLogPrerequisite(enablelog, statusdir);
//add all function arguments to a map
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
userArgs.put("command", command);
userArgs.put("optionsfile", optionsFile);
userArgs.put("libdir", libdir);
userArgs.put("files", otherFiles);
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
userArgs.put("enablejobreconnect", enablejobreconnect);
SqoopDelegator d = new SqoopDelegator(appConf);
return d.run(getDoAsUser(), userArgs, command, optionsFile, otherFiles,
statusdir, callback, getCompletedUrl(), enablelog, enablejobreconnect, libdir);
}
/**
* Run a Hive job.
* @param execute SQL statement to run, equivalent to "-e" from hive command line
* @param srcFile name of hive script file to run, equivalent to "-f" from hive
* command line
* @param hiveArgs additional command line argument passed to the hive command line.
* Please check https://cwiki.apache.org/Hive/languagemanual-cli.html
* for detailed explanation of command line arguments
* @param otherFiles additional files to be shipped to the launcher, such as the jars
* used in "add jar" statement in hive script
* @param defines shortcut for command line arguments "--define"
* @param statusdir where the stderr/stdout of templeton controller job goes
* @param callback URL which WebHCat will call when the hive job finishes
* @param enablelog whether to collect mapreduce log into statusdir/logs
* @param enablejobreconnect whether to reconnect to a running child job on templeton
* controller job retry
*/
@POST
@Path("hive")
@Produces({MediaType.APPLICATION_JSON})
public EnqueueBean hive(@FormParam("execute") String execute,
@FormParam("file") String srcFile,
@FormParam("arg") List<String> hiveArgs,
@FormParam("files") String otherFiles,
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
@FormParam("enablelog") boolean enablelog,
@FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException, TooManyRequestsException {
verifyUser();
if (execute == null && srcFile == null) {
throw new BadParam("Either execute or file parameter required");
}
//add all function arguments to a map
Map<String, Object> userArgs = new HashMap<String, Object>();
userArgs.put("user.name", getDoAsUser());
userArgs.put("execute", execute);
userArgs.put("file", srcFile);
userArgs.put("define", defines);
userArgs.put("files", otherFiles);
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
HiveDelegator d = new HiveDelegator(appConf);
return d.run(getDoAsUser(), userArgs, execute, srcFile, defines, hiveArgs, otherFiles,
statusdir, callback, getCompletedUrl(), enablelog, enablejobreconnect);
}
/**
* Return the status of the jobid.
*/
@GET
@Path("jobs/{jobid}")
@Produces({MediaType.APPLICATION_JSON})
public QueueStatusBean showJobId(@PathParam("jobid") String jobid)
throws NotAuthorizedException, BadParam, IOException, InterruptedException,
BusyException, TimeoutException, ExecutionException, TooManyRequestsException {
verifyUser();
verifyParam(jobid, ":jobid");
StatusDelegator d = new StatusDelegator(appConf);
return d.run(getDoAsUser(), jobid);
}
/**
* Kill a job in the queue.
*/
@DELETE
@Path("jobs/{jobid}")
@Produces({MediaType.APPLICATION_JSON})
public QueueStatusBean deleteJobId(@PathParam("jobid") String jobid)
throws NotAuthorizedException, BadParam, IOException, InterruptedException {
verifyUser();
verifyParam(jobid, ":jobid");
DeleteDelegator d = new DeleteDelegator(appConf);
return d.run(getDoAsUser(), jobid);
}
/**
* Return all the known job ids for this user based on the optional filter conditions.
* <p>
* Example usages:
* 1. curl -s 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan'
* Return all the Job IDs submitted by hsubramaniyan
* 2. curl -s
* 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan%26showall=true'
* Return all the Job IDs that are visible to hsubramaniyan
* 3. curl -s
* 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan%26jobid=job_201312091733_0003'
* Return all the Job IDs for hsubramaniyan after job_201312091733_0003.
* 4. curl -s 'http://localhost:50111/templeton/v1/jobs?
* user.name=hsubramaniyan%26jobid=job_201312091733_0003%26numrecords=5'
* Return the first 5(atmost) Job IDs submitted by hsubramaniyan after job_201312091733_0003.
* 5. curl -s
* 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan%26numrecords=5'
* Return the first 5(atmost) Job IDs submitted by hsubramaniyan after sorting the Job ID list
* lexicographically.
* </p>
* <p>
* Supporting pagination using "jobid" and "numrecords" parameters:
* Step 1: Get the start "jobid" = job_xxx_000, "numrecords" = n
* Step 2: Issue a curl command by specifying the user-defined "numrecords" and "jobid"
* Step 3: If list obtained from Step 2 has size equal to "numrecords", retrieve the list's
* last record and get the Job Id of the last record as job_yyy_k, else quit.
* Step 4: set "jobid"=job_yyy_k and go to step 2.
* </p>
* @param fields If "fields" set to "*", the request will return full details of the job.
* If "fields" is missing, will only return the job ID. Currently the value can only
* be "*", other values are not allowed and will throw exception.
* @param showall If "showall" is set to "true", the request will return all jobs the user
* has permission to view, not only the jobs belonging to the user.
* @param jobid If "jobid" is present, the records whose Job Id is lexicographically greater
* than "jobid" are only returned. For example, if "jobid" = "job_201312091733_0001",
* the jobs whose Job ID is greater than "job_201312091733_0001" are returned. The number of
* records returned depends on the value of "numrecords".
* @param numrecords If the "jobid" and "numrecords" parameters are present, the top #numrecords
* records appearing after "jobid" will be returned after sorting the Job Id list
* lexicographically.
* If "jobid" parameter is missing and "numrecords" is present, the top #numrecords will
* be returned after lexicographically sorting the Job Id list. If "jobid" parameter is present
* and "numrecords" is missing, all the records whose Job Id is greater than "jobid" are returned.
* @return list of job items based on the filter conditions specified by the user.
*/
@GET
@Path("jobs")
@Produces({MediaType.APPLICATION_JSON})
public List<JobItemBean> showJobList(@QueryParam("fields") String fields,
@QueryParam("showall") boolean showall,
@QueryParam("jobid") String jobid,
@QueryParam("numrecords") String numrecords)
throws NotAuthorizedException, BadParam, IOException, InterruptedException,
BusyException, TimeoutException, ExecutionException, TooManyRequestsException {
verifyUser();
boolean showDetails = false;
if (fields!=null && !fields.equals("*")) {
throw new BadParam("fields value other than * is not supported");
}
if (fields!=null && fields.equals("*")) {
showDetails = true;
}
int numRecords;
// Parse numrecords to an integer
try {
if (numrecords != null) {
numRecords = Integer.parseInt(numrecords);
if (numRecords <= 0) {
throw new BadParam("numrecords should be an integer > 0");
}
}
else {
numRecords = -1;
}
}
catch(Exception e) {
throw new BadParam("Invalid numrecords format: numrecords should be an integer > 0");
}
ListDelegator ld = new ListDelegator(appConf);
return ld.run(getDoAsUser(), showall, jobid, numRecords, showDetails);
}
/**
* Notify on a completed job. Called by JobTracker.
*/
@GET
@Path("internal/complete/{jobid}")
@Produces({MediaType.APPLICATION_JSON})
public CompleteBean completeJob(@PathParam("jobid") String jobid,
@QueryParam("status") String jobStatus)
throws CallbackFailedException, IOException {
LOG.debug("Received callback " + theUriInfo.getRequestUri());
CompleteDelegator d = new CompleteDelegator(appConf);
return d.run(jobid, jobStatus);
}
/**
* Verify that we have a valid user. Throw an exception if invalid.
*/
public void verifyUser() throws NotAuthorizedException {
String requestingUser = getRequestingUser();
if (requestingUser == null) {
String msg = "No user found.";
if (!UserGroupInformation.isSecurityEnabled()) {
msg += " Missing " + PseudoAuthenticator.USER_NAME + " parameter.";
}
throw new NotAuthorizedException(msg);
}
if(doAs != null && !doAs.equals(requestingUser)) {
/*if doAs user is different than logged in user, need to check that
that logged in user is authorized to run as 'doAs'*/
ProxyUserSupport.validate(requestingUser, getRequestingHost(requestingUser, request), doAs);
}
}
/**
* All 'tasks' spawned by WebHCat should be run as this user. W/o doAs query parameter
* this is just the user making the request (or
* {@link org.apache.hadoop.security.authentication.client.PseudoAuthenticator#USER_NAME}
* query param).
* @return value of doAs query parameter or {@link #getRequestingUser()}
*/
private String getDoAsUser() {
return doAs != null && !doAs.equals(getRequestingUser()) ? doAs : getRequestingUser();
}
/**
* Verify that the parameter exists. Throw an exception if invalid.
*/
public void verifyParam(String param, String name)
throws BadParam {
if (param == null) {
throw new BadParam("Missing " + name + " parameter");
}
}
/**
* Verify that the parameter exists. Throw an exception if invalid.
*/
public void verifyParam(List<String> param, String name)
throws BadParam {
if (param == null || param.isEmpty()) {
throw new BadParam("Missing " + name + " parameter");
}
}
public static final Pattern DDL_ID = Pattern.compile("[a-zA-Z]\\w*");
public static final Pattern PROPERTY_ID =
Pattern.compile("[a-zA-Z0-9][\\w\\.\\-]*(?<!\\-)(?<!\\.)(?<!\\_)$");
/**
* Verify that the parameter exists and is a simple DDL identifier
* name. Throw an exception if invalid.
*
* Bug: This needs to allow for quoted ddl identifiers.
*/
public void verifyDdlParam(String param, String name)
throws BadParam {
verifyParam(param, name);
Matcher m = DDL_ID.matcher(param);
if (!m.matches()) {
throw new BadParam("Invalid DDL identifier " + name);
}
}
/**
* Verify that the parameter exists and is a valid property
* name. Throw an exception if invalid.
*
*/
public void verifyPropertyParam(String param, String name)
throws BadParam {
verifyParam(param, name);
Matcher m = PROPERTY_ID.matcher(param);
if (!m.matches()) {
throw new BadParam("Invalid property name " + name);
}
}
/**
* Get the user name from the security context, i.e. the user making the HTTP request.
* With simple/pseudo security mode this should return the
* value of user.name query param, in kerberos mode it's the kinit'ed user.
*/
private String getRequestingUser() {
if (theSecurityContext == null) {
return null;
}
String userName = null;
if (theSecurityContext.getUserPrincipal() == null) {
userName = Main.UserNameHandler.getUserName(request);
}
else {
userName = theSecurityContext.getUserPrincipal().getName();
}
if(userName == null) {
return null;
}
//map hue/foo.bar@something.com->hue since user group checks
// and config files are in terms of short name
return UserGroupInformation.createRemoteUser(userName).getShortUserName();
}
/**
* The callback url on this server when a task is completed.
*/
public String getCompletedUrl() {
if (theUriInfo == null) {
return null;
}
if (theUriInfo.getBaseUri() == null) {
return null;
}
return theUriInfo.getBaseUri() + VERSION
+ "/internal/complete/$jobId?status=$jobStatus";
}
/**
* Returns canonical host name from which the request is made; used for doAs validation
*/
private static String getRequestingHost(String requestingUser, HttpServletRequest request) {
final String unkHost = "???";
if(request == null) {
LOG.warn("request is null; cannot determine hostname");
return unkHost;
}
try {
String address = request.getRemoteAddr();//returns IP addr
if(address == null) {
LOG.warn(MessageFormat.format("Request remote address is NULL for user [{0}]", requestingUser));
return unkHost;
}
//Inet4Address/Inet6Address
String hostName = InetAddress.getByName(address).getCanonicalHostName();
if(LOG.isDebugEnabled()) {
LOG.debug(MessageFormat.format("Resolved remote hostname: [{0}]", hostName));
}
return hostName;
} catch (UnknownHostException ex) {
LOG.warn(MessageFormat.format("Request remote address could not be resolved, {0}", ex.toString(), ex));
return unkHost;
}
}
private void checkEnableLogPrerequisite(boolean enablelog, String statusdir) throws BadParam {
if (enablelog && !TempletonUtils.isset(statusdir))
throw new BadParam("enablelog is only applicable when statusdir is set");
if(enablelog && "0.23".equalsIgnoreCase(ShimLoader.getMajorVersion())) {
throw new BadParam("enablelog=true is only supported with Hadoop 1.x");
}
}
}