/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.conf;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hive.common.util.HiveStringUtils;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
/**
* Hive Configuration utils
*/
@Private
public class HiveConfUtil {
private static final String CLASS_NAME = HiveConfUtil.class.getName();
private static final Log LOG = LogFactory.getLog(CLASS_NAME);
/**
* Check if metastore is being used in embedded mode.
* This utility function exists so that the logic for determining the mode is same
* in HiveConf and HiveMetaStoreClient
* @param msUri - metastore server uri
* @return
*/
public static boolean isEmbeddedMetaStore(String msUri) {
return (msUri == null) ? true : msUri.trim().isEmpty();
}
/**
* Dumps all HiveConf for debugging. Convenient to dump state at process start up and log it
* so that in later analysis the values of all variables is known
*/
public static StringBuilder dumpConfig(HiveConf conf) {
StringBuilder sb = new StringBuilder("START========\"HiveConf()\"========\n");
sb.append("hiveDefaultUrl=").append(conf.getHiveDefaultLocation()).append('\n');
sb.append("hiveSiteURL=").append(HiveConf.getHiveSiteLocation()).append('\n');
sb.append("hiveServer2SiteUrl=").append(HiveConf.getHiveServer2SiteLocation()).append('\n');
sb.append("hivemetastoreSiteUrl=").append(HiveConf.getMetastoreSiteLocation()).append('\n');
dumpConfig(conf, sb);
return sb.append("END========\"new HiveConf()\"========\n");
}
/**
* Getting the set of the hidden configurations
* @param configuration The original configuration
* @return The list of the configuration values to hide
*/
public static Set<String> getHiddenSet(Configuration configuration) {
Set<String> hiddenSet = new HashSet<String>();
String hiddenListStr = HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST);
if (hiddenListStr != null) {
for (String entry : hiddenListStr.split(",")) {
hiddenSet.add(entry.trim());
}
}
return hiddenSet;
}
/**
* Strips hidden config entries from configuration
* @param conf The configuration to strip from
* @param hiddenSet The values to strip
*/
public static void stripConfigurations(Configuration conf, Set<String> hiddenSet) {
for (String name : hiddenSet) {
if (conf.get(name) != null) {
conf.set(name, StringUtils.EMPTY);
}
}
}
/**
* Searches the given configuration object and replaces all the configuration values for keys
* defined hive.conf.hidden.list by empty String
*
* @param conf - Configuration object which needs to be modified to remove sensitive keys
*/
public static void stripConfigurations(Configuration conf) {
Set<String> hiddenSet = getHiddenSet(conf);
stripConfigurations(conf, hiddenSet);
}
public static void dumpConfig(Configuration originalConf, StringBuilder sb) {
Set<String> hiddenSet = getHiddenSet(originalConf);
sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n");
Configuration conf = new Configuration(originalConf);
stripConfigurations(conf, hiddenSet);
Iterator<Map.Entry<String, String>> configIter = conf.iterator();
List<Map.Entry<String, String>> configVals = new ArrayList<>();
while(configIter.hasNext()) {
configVals.add(configIter.next());
}
Collections.sort(configVals, new Comparator<Map.Entry<String, String>>() {
@Override
public int compare(Map.Entry<String, String> ent, Map.Entry<String, String> ent2) {
return ent.getKey().compareTo(ent2.getKey());
}
});
for(Map.Entry<String, String> entry : configVals) {
//use get() to make sure variable substitution works
if(entry.getKey().toLowerCase().contains("path")) {
StringTokenizer st = new StringTokenizer(conf.get(entry.getKey()), File.pathSeparator);
sb.append(entry.getKey()).append("=\n");
while(st.hasMoreTokens()) {
sb.append(" ").append(st.nextToken()).append(File.pathSeparator).append('\n');
}
}
else {
sb.append(entry.getKey()).append('=').append(conf.get(entry.getKey())).append('\n');
}
}
}
/**
* Updates the job configuration with the job specific credential provider information available
* in the HiveConf.It uses the environment variables HADOOP_CREDSTORE_PASSWORD or
* HIVE_JOB_CREDSTORE_PASSWORD to get the custom password for all the keystores configured in the
* provider path. This usage of environment variables is similar in lines with Hadoop credential
* provider mechanism for getting the keystore passwords. The other way of communicating the
* password is through a file which stores the password in clear-text which needs to be readable
* by all the consumers and therefore is not supported.
*
*<ul>
* <li>If HIVE_SERVER2_JOB_CREDENTIAL_PROVIDER_PATH is set in the hive configuration this method
* overrides the MR job configuration property hadoop.security.credential.provider.path with its
* value. If not set then it does not change the value of hadoop.security.credential.provider.path
* <li>In order to choose the password for the credential provider we check :
*
* (1) if job credential provider path HIVE_SERVER2_JOB_CREDENTIAL_PROVIDER_PATH is set we check if
* HIVE_SERVER2_JOB_CREDSTORE_PASSWORD_ENVVAR is set. If it is set we use it.
* (2) If password is not set using (1) above we use HADOOP_CREDSTORE_PASSWORD if it is set.
* (3) If none of those are set, we do not set any password in the MR task environment. In this
* case the hadoop credential provider should use the default password of "none" automatically
*</ul>
* @param jobConf - job specific configuration
*/
public static void updateJobCredentialProviders(Configuration jobConf) {
if(jobConf == null) {
return;
}
String jobKeyStoreLocation = jobConf.get(HiveConf.ConfVars.HIVE_SERVER2_JOB_CREDENTIAL_PROVIDER_PATH.varname);
String oldKeyStoreLocation = jobConf.get(Constants.HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG);
if (StringUtils.isNotBlank(jobKeyStoreLocation)) {
jobConf.set(Constants.HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG, jobKeyStoreLocation);
LOG.debug("Setting job conf credstore location to " + jobKeyStoreLocation
+ " previous location was " + oldKeyStoreLocation);
}
String credStorepassword = getJobCredentialProviderPassword(jobConf);
if (credStorepassword != null) {
// if the execution engine is MR set the map/reduce env with the credential store password
String execEngine = jobConf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname);
if ("mr".equalsIgnoreCase(execEngine)) {
addKeyValuePair(jobConf, JobConf.MAPRED_MAP_TASK_ENV,
Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword);
addKeyValuePair(jobConf, JobConf.MAPRED_REDUCE_TASK_ENV,
Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword);
addKeyValuePair(jobConf, "yarn.app.mapreduce.am.admin.user.env",
Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword);
}
}
}
/*
* If HIVE_SERVER2_JOB_CREDSTORE_LOCATION is set check HIVE_SERVER2_JOB_CREDSTORE_PASSWORD_ENVVAR before
* checking HADOOP_CREDENTIAL_PASSWORD_ENVVAR
*/
public static String getJobCredentialProviderPassword(Configuration conf) {
String jobKeyStoreLocation =
conf.get(HiveConf.ConfVars.HIVE_SERVER2_JOB_CREDENTIAL_PROVIDER_PATH.varname);
String password = null;
if(StringUtils.isNotBlank(jobKeyStoreLocation)) {
password = System.getenv(Constants.HIVE_SERVER2_JOB_CREDSTORE_PASSWORD_ENVVAR);
if (StringUtils.isNotBlank(password)) {
return password;
}
}
password = System.getenv(Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR);
if (StringUtils.isNotBlank(password)) {
return password;
}
return null;
}
private static void addKeyValuePair(Configuration jobConf, String property, String keyName,
String newKeyValue) {
String existingValue = jobConf.get(property);
if (existingValue == null) {
jobConf.set(property, (keyName + "=" + newKeyValue));
return;
}
String propertyValue = HiveStringUtils.insertValue(keyName, newKeyValue, existingValue);
jobConf.set(property, propertyValue);
}
}