/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ranger.plugin.client;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.SecureClientLogin;
public class HadoopConfigHolder {
private static final Log LOG = LogFactory.getLog(HadoopConfigHolder.class);
public static final String GLOBAL_LOGIN_PARAM_PROP_FILE = "hadoop-login.properties";
public static final String DEFAULT_DATASOURCE_PARAM_PROP_FILE = "datasource.properties";
public static final String RESOURCEMAP_PROP_FILE = "resourcenamemap.properties";
public static final String DEFAULT_RESOURCE_NAME = "core-site.xml";
public static final String RANGER_SECTION_NAME = "xalogin.xml";
public static final String RANGER_LOGIN_USER_NAME_PROP = "username";
public static final String RANGER_LOGIN_KEYTAB_FILE_PROP = "keytabfile";
public static final String RANGER_LOGIN_PASSWORD = "password";
public static final String RANGER_LOOKUP_PRINCIPAL = "lookupprincipal";
public static final String RANGER_LOOKUP_KEYTAB = "lookupkeytab";
public static final String RANGER_PRINCIPAL = "rangerprincipal";
public static final String RANGER_KEYTAB = "rangerkeytab";
public static final String RANGER_NAME_RULES = "namerules";
public static final String RANGER_AUTH_TYPE = "authtype";
public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication";
public static final String HADOOP_NAME_RULES = "hadoop.security.auth_to_local";
public static final String HADOOP_SECURITY_AUTHENTICATION_METHOD = "kerberos";
public static final String HADOOP_RPC_PROTECTION = "hadoop.rpc.protection";
public static final String ENABLE_HIVE_METASTORE_LOOKUP = "enable.hive.metastore.lookup";
public static final String HIVE_SITE_FILE_PATH = "hive.site.file.path";
private static boolean initialized;
private static Map<String,HashMap<String,Properties>> dataSource2ResourceListMap = new HashMap<>();
private static Map<String,HadoopConfigHolder> dataSource2HadoopConfigHolder = new HashMap<>();
private static Properties globalLoginProp = new Properties();
private static Properties resourcemapProperties;
private String datasourceName;
private String defaultConfigFile;
private String userName;
private String keyTabFile;
private String password;
private String lookupPrincipal;
private String lookupKeytab;
private String nameRules;
private String authType;
private String hiveSiteFilePath;
private boolean isKerberosAuth;
private boolean enableHiveMetastoreLookup;
private Map<String,String> connectionProperties;
private static Set<String> rangerInternalPropertyKeys = new HashSet<>();
public static HadoopConfigHolder getInstance(String aDatasourceName) {
HadoopConfigHolder ret = dataSource2HadoopConfigHolder.get(aDatasourceName);
if (ret == null) {
synchronized (HadoopConfigHolder.class) {
HadoopConfigHolder temp = ret;
if (temp == null) {
ret = new HadoopConfigHolder(aDatasourceName);
dataSource2HadoopConfigHolder.put(aDatasourceName, ret);
}
}
}
return ret;
}
public static HadoopConfigHolder getInstance(String aDatasourceName, Map<String,String> connectionProperties) {
return getInstance(aDatasourceName, connectionProperties, null);
}
public static HadoopConfigHolder getInstance(String aDatasourceName, Map<String,String> connectionProperties,
String defaultConfigFile) {
HadoopConfigHolder ret = dataSource2HadoopConfigHolder.get(aDatasourceName);
if (ret == null) {
synchronized (HadoopConfigHolder.class) {
HadoopConfigHolder temp = ret;
if (temp == null) {
ret = new HadoopConfigHolder(aDatasourceName,connectionProperties, defaultConfigFile);
dataSource2HadoopConfigHolder.put(aDatasourceName, ret);
}
}
}
else {
if (connectionProperties !=null && !connectionProperties.equals(ret.connectionProperties)) {
ret = new HadoopConfigHolder(aDatasourceName,connectionProperties);
dataSource2HadoopConfigHolder.remove(aDatasourceName);
dataSource2HadoopConfigHolder.put(aDatasourceName, ret);
}
}
return ret;
}
private HadoopConfigHolder(String aDatasourceName) {
datasourceName = aDatasourceName;
if (!initialized) {
init();
}
initLoginInfo();
}
private HadoopConfigHolder(String aDatasourceName,
Map<String, String> connectionProperties) {
this(aDatasourceName, connectionProperties, null);
}
private HadoopConfigHolder(String aDatasourceName, Map<String,String> connectionProperties,
String defaultConfigFile) {
datasourceName = aDatasourceName;
this.connectionProperties = connectionProperties;
this.defaultConfigFile = defaultConfigFile;
initConnectionProp();
initLoginInfo();
}
private void initConnectionProp() {
if (!connectionProperties.containsKey(ENABLE_HIVE_METASTORE_LOOKUP)) {
connectionProperties.put(ENABLE_HIVE_METASTORE_LOOKUP, "false");
}
if (!connectionProperties.containsKey(HIVE_SITE_FILE_PATH)) {
connectionProperties.put(HIVE_SITE_FILE_PATH, "");
}
for (Map.Entry<String, String> entry : connectionProperties.entrySet()) {
String key = entry.getKey();
String resourceName = getResourceName(key);
if (resourceName == null) {
resourceName = RANGER_SECTION_NAME;
}
String val = entry.getValue();
addConfiguration(datasourceName, resourceName, key, val );
}
}
private String getResourceName(String key) {
if (resourcemapProperties == null) {
initResourceMap();
}
if (resourcemapProperties != null) {
String rn = resourcemapProperties.getProperty(key);
return ( rn != null) ? rn : defaultConfigFile;
} else {
return defaultConfigFile;
}
}
public static void initResourceMap() {
if (resourcemapProperties == null) {
resourcemapProperties = new Properties();
InputStream in = HadoopConfigHolder.class.getClassLoader().getResourceAsStream(RESOURCEMAP_PROP_FILE);
if (in != null) {
try {
resourcemapProperties.load(in);
for (Map.Entry<Object, Object> entry : resourcemapProperties
.entrySet()) {
String key = (String) entry.getKey();
String value = (String) entry.getValue();
if (RANGER_SECTION_NAME.equals(value)) {
rangerInternalPropertyKeys.add(key);
}
}
} catch (IOException e) {
throw new HadoopException("Unable to load resource map properties from [" + RESOURCEMAP_PROP_FILE + "]", e);
}
finally {
if (in != null) {
try {
in.close();
} catch (IOException ioe) {
// Ignore IOException during close of stream
}
}
}
} else {
throw new HadoopException("Unable to locate resource map properties from [" + RESOURCEMAP_PROP_FILE + "] in the class path.");
}
}
}
private static synchronized void init() {
if (initialized) {
return;
}
try {
InputStream in = HadoopConfigHolder.class.getClassLoader().getResourceAsStream(DEFAULT_DATASOURCE_PARAM_PROP_FILE);
if (in != null) {
Properties prop = new Properties();
try {
prop.load(in);
} catch (IOException e) {
throw new HadoopException("Unable to get configuration information for Hadoop environments", e);
}
finally {
try {
in.close();
} catch (IOException e) {
// Ignored exception when the stream is closed.
}
}
if (prop.isEmpty()) {
return;
}
for (Object keyobj : prop.keySet()) {
String key = (String)keyobj;
String val = prop.getProperty(key);
int dotLocatedAt = key.indexOf(".");
if (dotLocatedAt == -1) {
continue;
}
String dataSource = key.substring(0,dotLocatedAt);
String propKey = key.substring(dotLocatedAt+1);
int resourceFoundAt = propKey.indexOf(".");
if (resourceFoundAt > -1) {
String resourceName = propKey.substring(0, resourceFoundAt) + ".xml";
propKey = propKey.substring(resourceFoundAt+1);
addConfiguration(dataSource, resourceName, propKey, val);
}
}
}
in = HadoopConfigHolder.class.getClassLoader().getResourceAsStream(GLOBAL_LOGIN_PARAM_PROP_FILE);
if (in != null) {
Properties tempLoginProp = new Properties();
try {
tempLoginProp.load(in);
} catch (IOException e) {
throw new HadoopException("Unable to get login configuration information for Hadoop environments from file: [" + GLOBAL_LOGIN_PARAM_PROP_FILE + "]", e);
}
finally {
try {
in.close();
} catch (IOException e) {
// Ignored exception when the stream is closed.
}
}
globalLoginProp = tempLoginProp;
}
}
finally {
initialized = true;
}
}
private void initLoginInfo() {
Properties prop = this.getRangerSection();
if (prop != null) {
userName = prop.getProperty(RANGER_LOGIN_USER_NAME_PROP);
keyTabFile = prop.getProperty(RANGER_LOGIN_KEYTAB_FILE_PROP);
if (StringUtils.trimToNull(prop.getProperty(ENABLE_HIVE_METASTORE_LOOKUP)) != null) {
try {
enableHiveMetastoreLookup = Boolean.valueOf(prop.getProperty(ENABLE_HIVE_METASTORE_LOOKUP,"false").trim());
} catch (Exception e) {
enableHiveMetastoreLookup = false;
LOG.error("Error while getting " + ENABLE_HIVE_METASTORE_LOOKUP + " : " + e.getMessage());
}
}
if (StringUtils.trimToNull(prop.getProperty(HIVE_SITE_FILE_PATH)) != null) {
hiveSiteFilePath = prop.getProperty(HIVE_SITE_FILE_PATH).trim();
} else {
hiveSiteFilePath = null;
}
password = prop.getProperty(RANGER_LOGIN_PASSWORD);
lookupPrincipal = prop.getProperty(RANGER_LOOKUP_PRINCIPAL);
lookupKeytab = prop.getProperty(RANGER_LOOKUP_KEYTAB);
nameRules = prop.getProperty(RANGER_NAME_RULES);
authType = prop.getProperty(RANGER_AUTH_TYPE, "simple");
String hadoopSecurityAuthentication = getHadoopSecurityAuthentication();
if (hadoopSecurityAuthentication != null) {
isKerberosAuth = HADOOP_SECURITY_AUTHENTICATION_METHOD.equalsIgnoreCase(hadoopSecurityAuthentication);
} else {
isKerberosAuth = (((userName != null) && (userName.indexOf("@") > -1)) || (SecureClientLogin.isKerberosCredentialExists(lookupPrincipal, lookupKeytab)));
}
}
}
public Properties getRangerSection() {
Properties prop = this.getProperties(RANGER_SECTION_NAME);
if (prop == null) {
prop = globalLoginProp;
}
return prop;
}
private static void addConfiguration(String dataSource, String resourceName, String propertyName, String value) {
if (dataSource == null || dataSource.isEmpty()) {
return;
}
if (propertyName == null || propertyName.isEmpty()) {
return;
}
if (resourceName == null) {
resourceName = DEFAULT_RESOURCE_NAME;
}
HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(dataSource);
if (resourceName2PropertiesMap == null) {
resourceName2PropertiesMap = new HashMap<>();
dataSource2ResourceListMap.put(dataSource, resourceName2PropertiesMap);
}
Properties prop = resourceName2PropertiesMap.get(resourceName);
if (prop == null) {
prop = new Properties();
resourceName2PropertiesMap.put(resourceName, prop);
}
if (value == null) {
prop.remove(propertyName);
} else {
prop.put(propertyName, value);
}
}
public String getDatasourceName() {
return datasourceName;
}
public boolean hasResourceExists(String aResourceName) { // dilli
HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(datasourceName);
return (resourceName2PropertiesMap != null && resourceName2PropertiesMap.containsKey(aResourceName));
}
public Properties getProperties(String aResourceName) {
Properties ret = null;
HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(datasourceName);
if (resourceName2PropertiesMap != null) {
ret = resourceName2PropertiesMap.get(aResourceName);
}
return ret;
}
public String getHadoopSecurityAuthentication() {
String ret = null;
String sectionName = RANGER_SECTION_NAME;
if (defaultConfigFile != null) {
sectionName = defaultConfigFile;
}
if (LOG.isDebugEnabled()) {
LOG.debug("==> HadoopConfigHolder.getHadoopSecurityAuthentication( " + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + ")" );
}
ret = getProperties(sectionName,HADOOP_SECURITY_AUTHENTICATION);
if (LOG.isDebugEnabled()) {
LOG.debug("<== HadoopConfigHolder.getHadoopSecurityAuthentication(" + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + " Value : " + ret + ")" );
}
return ret;
}
public String getUserName() {
return userName;
}
public String getKeyTabFile() {
return keyTabFile;
}
public String getPassword() {
return password;
}
public boolean isKerberosAuthentication() {
return isKerberosAuth;
}
public String getLookupPrincipal() {
return lookupPrincipal;
}
public String getLookupKeytab() {
return lookupKeytab;
}
public String getNameRules() {
return nameRules;
}
public String getAuthType() {
return authType;
}
public boolean isEnableHiveMetastoreLookup() {
return enableHiveMetastoreLookup;
}
public String getHiveSiteFilePath() {
return hiveSiteFilePath;
}
public Set<String> getRangerInternalPropertyKeys() {
return rangerInternalPropertyKeys;
}
private String getProperties(String sectionName, String property) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + ")" );
}
Properties repoParam = null;
String ret = null;
HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(this.getDatasourceName());
if (resourceName2PropertiesMap != null) {
repoParam=resourceName2PropertiesMap.get(sectionName);
}
if (repoParam != null) {
ret = (String)repoParam.get(property);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + " Value : " + ret);
}
return ret;
}
}