/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.falcon.entity;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.datasource.Credential;
import org.apache.falcon.entity.v0.datasource.Credentialtype;
import org.apache.falcon.entity.v0.datasource.Datasource;
import org.apache.falcon.entity.v0.datasource.DatasourceType;
import org.apache.falcon.entity.v0.datasource.Driver;
import org.apache.falcon.entity.v0.datasource.Interface;
import org.apache.falcon.entity.v0.datasource.Interfaces;
import org.apache.falcon.entity.v0.datasource.Interfacetype;
import org.apache.falcon.entity.v0.datasource.PasswordAliasType;
import org.apache.falcon.entity.v0.datasource.Property;
import org.apache.falcon.security.CurrentUser;
import org.apache.hadoop.conf.Configuration;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.security.CredentialProviderHelper;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.Map;
/**
* DataSource entity helper methods.
*/
public final class DatasourceHelper {
public static final String HADOOP_CREDENTIAL_PROVIDER_FILEPATH = "hadoop.security.credential.provider.path";
private static final Logger LOG = LoggerFactory.getLogger(DatasourceHelper.class);
private static final ConfigurationStore STORE = ConfigurationStore.get();
public static DatasourceType getDatasourceType(String datasourceName) throws FalconException {
return getDatasource(datasourceName).getType();
}
private DatasourceHelper() {}
public static Datasource getDatasource(String datasourceName) throws FalconException {
return STORE.get(EntityType.DATASOURCE, datasourceName);
}
public static String getReadOnlyEndpoint(Datasource datasource) {
return getInterfaceEndpoint(datasource, Interfacetype.READONLY);
}
public static String getWriteEndpoint(Datasource datasource) {
return getInterfaceEndpoint(datasource, Interfacetype.WRITE);
}
/**
* Returns user name and password pair as it is specified in the XML. If the credential type is
* password-file, the path name is returned.
*
* @param db
* @return Credential
* @throws FalconException
*/
public static Credential getReadPasswordInfo(Datasource db) throws FalconException {
for (Interface ifs : db.getInterfaces().getInterfaces()) {
if ((ifs.getType() == Interfacetype.READONLY) && (ifs.getCredential() != null)) {
return ifs.getCredential();
}
}
return getDefaultPasswordInfo(db.getInterfaces());
}
public static Credential getWritePasswordInfo(Datasource db) throws FalconException {
for (Interface ifs : db.getInterfaces().getInterfaces()) {
if ((ifs.getType() == Interfacetype.WRITE) && (ifs.getCredential() != null)) {
return ifs.getCredential();
}
}
return getDefaultPasswordInfo(db.getInterfaces());
}
/**
* Returns user name and actual password pair. If the credential type is password-file, then the
* password is read from the HDFS file. If the credential type is password-text, the clear text
* password is returned.
*
* @param db
* @return
* @throws FalconException
*/
public static java.util.Properties fetchReadPasswordInfo(Datasource db) throws FalconException {
Credential cred = getReadPasswordInfo(db);
return fetchPasswordInfo(cred);
}
public static java.util.Properties fetchWritePasswordInfo(Datasource db) throws FalconException {
Credential cred = getWritePasswordInfo(db);
return fetchPasswordInfo(cred);
}
public static java.util.Properties fetchPasswordInfo(Credential cred) throws FalconException {
java.util.Properties p = new java.util.Properties();
p.put("user", cred.getUserName());
if (cred.getType() == Credentialtype.PASSWORD_TEXT) {
p.put("password", cred.getPasswordText());
} else if (cred.getType() == Credentialtype.PASSWORD_FILE) {
String actualPasswd = fetchPasswordInfoFromFile(cred.getPasswordFile());
p.put("password", actualPasswd);
} else if (cred.getType() == Credentialtype.PASSWORD_ALIAS) {
String actualPasswd = fetchPasswordInfoFromCredentialStore(cred.getPasswordAlias());
p.put("password", actualPasswd);
}
return p;
}
public static String buildJceksProviderPath(URI credURI) {
StringBuilder sb = new StringBuilder();
final String credProviderPath = sb.append("jceks:").append("//")
.append(credURI.getScheme()).append("@")
.append(credURI.getHost())
.append(credURI.getPath()).toString();
return credProviderPath;
}
/**
* checks if two datasource interfaces are same.
*
* @param oldEntity old datasource entity
* @param newEntity new datasource entity
* @param ifacetype type of interface
* @return true if same else false
*/
public static boolean isSameInterface(Datasource oldEntity, Datasource newEntity, Interfacetype ifacetype) {
LOG.debug("Verifying if Interfaces match for Datasource {} : Old - {}, New - {}", oldEntity, newEntity);
Interface oIface = getInterface(oldEntity, ifacetype);
Interface nIface = getInterface(newEntity, ifacetype);
if ((oIface == null) && (nIface == null)) {
return true;
}
if ((oIface == null) || (nIface == null)) {
return false;
}
return (StringUtils.equals(oIface.getEndpoint(), nIface.getEndpoint())
&& isSameDriverClazz(oIface.getDriver(), nIface.getDriver())
&& isSameCredentials(oIface.getCredential(), nIface.getCredential()));
}
/**
* check if datasource driver is same.
* @param oldEntity
* @param newEntity
* @return true if same or false
*/
public static boolean isSameDriverClazz(Driver oldEntity, Driver newEntity) {
if ((oldEntity == null) && (newEntity == null)) {
return true;
}
if ((oldEntity == null) || (newEntity == null)) {
return false;
}
return StringUtils.equals(oldEntity.getClazz(), newEntity.getClazz());
}
/**
* checks if data source properties are same.
* @param oldEntity
* @param newEntity
* @return true if same else false
*/
public static boolean isSameProperties(Datasource oldEntity, Datasource newEntity) {
Map<String, String> oldProps = getDatasourceProperties(oldEntity);
Map<String, String> newProps = getDatasourceProperties(newEntity);
return oldProps.equals(newProps);
}
/**
* checks if data source credentials are same.
* @param oCred
* @param nCred
* @return true true
*/
public static boolean isSameCredentials(Credential oCred, Credential nCred) {
if ((oCred == null) && (nCred == null)) {
return true;
}
if ((oCred == null) || (nCred == null)) {
return true;
}
if (StringUtils.equals(oCred.getUserName(), nCred.getUserName())) {
if (oCred.getType() == nCred.getType()) {
if (oCred.getType() == Credentialtype.PASSWORD_TEXT) {
return StringUtils.equals(oCred.getPasswordText(), nCred.getPasswordText());
} else if (oCred.getType() == Credentialtype.PASSWORD_FILE) {
return StringUtils.equals(oCred.getPasswordFile(), nCred.getPasswordFile());
} else if (oCred.getType() == Credentialtype.PASSWORD_ALIAS) {
return (StringUtils.equals(oCred.getPasswordAlias().getAlias(),
nCred.getPasswordAlias().getAlias())
&& StringUtils.equals(oCred.getPasswordAlias().getProviderPath(),
nCred.getPasswordAlias().getProviderPath()));
}
} else {
return false;
}
}
return false;
}
public static Credential getCredential(Datasource db) {
return getCredential(db, null);
}
public static Credential getCredential(Datasource db, Interfacetype interfaceType) {
if (interfaceType == null) {
return db.getInterfaces().getCredential();
} else {
for(Interface iface : db.getInterfaces().getInterfaces()) {
if (iface.getType() == interfaceType) {
return iface.getCredential();
}
}
}
return null;
}
public static void validateCredential(Credential cred) throws FalconException {
if (cred == null) {
return;
}
switch (cred.getType()) {
case PASSWORD_TEXT:
if (StringUtils.isBlank(cred.getUserName()) || StringUtils.isBlank(cred.getPasswordText())) {
throw new FalconException(String.format("Credential type '%s' missing tags '%s' or '%s'",
cred.getType().value(), "userName", "passwordText"));
}
break;
case PASSWORD_FILE:
if (StringUtils.isBlank(cred.getUserName()) || StringUtils.isBlank(cred.getPasswordFile())) {
throw new FalconException(String.format("Credential type '%s' missing tags '%s' or '%s'",
cred.getType().value(), "userName", "passwordFile"));
}
break;
case PASSWORD_ALIAS:
if (StringUtils.isBlank(cred.getUserName()) || (cred.getPasswordAlias() == null)
|| StringUtils.isBlank(cred.getPasswordAlias().getAlias())
|| StringUtils.isBlank(cred.getPasswordAlias().getProviderPath())) {
throw new FalconException(String.format("Credential type '%s' missing tags '%s' or '%s' or %s'",
cred.getType().value(), "userName", "alias", "providerPath"));
}
break;
default:
throw new FalconException(String.format("Unknown Credential type '%s'", cred.getType().value()));
}
}
/**
* Return the Interface endpoint for the interface type specified in the argument.
*
* @param db
* @param type - can be read-only or write
* @return
*/
private static String getInterfaceEndpoint(Datasource db, Interfacetype type) {
if (getInterface(db, type) != null) {
return getInterface(db, type).getEndpoint();
} else {
return null;
}
}
private static Interface getInterface(Datasource db, Interfacetype type) {
for(Interface ifs : db.getInterfaces().getInterfaces()) {
if (ifs.getType() == type) {
return ifs;
}
}
return null;
}
private static Credential getDefaultPasswordInfo(Interfaces ifs) throws FalconException {
if (ifs.getCredential() != null) {
return ifs.getCredential();
} else {
throw new FalconException("Missing Interfaces default credential");
}
}
/**
* fetch password from the corresponding store.
* @param c
* @return actual password
* @throws FalconException
*/
private static String fetchPasswordInfoFromCredentialStore(final PasswordAliasType c) throws FalconException {
try {
final String credPath = c.getProviderPath();
final URI credURI = new URI(credPath);
if (StringUtils.isBlank(credURI.getScheme())
|| StringUtils.isBlank(credURI.getHost())
|| StringUtils.isBlank(credURI.getPath())) {
throw new FalconException("Password alias jceks provider HDFS path is incorrect.");
}
final String alias = c.getAlias();
if (StringUtils.isBlank(alias)) {
throw new FalconException("Password alias is empty.");
}
final String credProviderPath = buildJceksProviderPath(credURI);
LOG.info("Credential provider HDFS path : " + credProviderPath);
if (CredentialProviderHelper.isProviderAvailable()) {
UserGroupInformation ugi = CurrentUser.getProxyUGI();
String password = ugi.doAs(new PrivilegedExceptionAction<String>() {
public String run() throws Exception {
final Configuration conf = new Configuration();
conf.set(HadoopClientFactory.FS_DEFAULT_NAME_KEY, credPath);
conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, credProviderPath);
FileSystem fs = FileSystem.get(credURI, conf);
if (!fs.exists(new Path(credPath))) {
String msg = String.format("Credential provider hdfs path [%s] does not "
+ "exist or access denied!", credPath);
LOG.error(msg);
throw new FalconException(msg);
}
return CredentialProviderHelper.resolveAlias(conf, alias);
}
});
return password;
} else {
throw new FalconException("Credential Provider is not initialized");
}
} catch (Exception ioe) {
String msg = "Exception while trying to fetch credential alias";
LOG.error(msg, ioe);
throw new FalconException(msg, ioe);
}
}
/**
* fetch the password from file.
*
* @param passwordFilePath
* @return
* @throws FalconException
*/
private static String fetchPasswordInfoFromFile(String passwordFilePath) throws FalconException {
try {
Path path = new Path(passwordFilePath);
FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri());
if (!fs.exists(path)) {
throw new IOException("The password file does not exist! ");
}
if (!fs.isFile(path)) {
throw new IOException("The password file cannot be a directory! ");
}
InputStream is = fs.open(path);
StringWriter writer = new StringWriter();
try {
IOUtils.copy(is, writer);
return writer.toString();
} finally {
IOUtils.closeQuietly(is);
IOUtils.closeQuietly(writer);
fs.close();
}
} catch (IOException ioe) {
LOG.error("Error reading password file from HDFS : " + ioe);
throw new FalconException(ioe);
}
}
/*
returns data store properties
*/
public static Map<String, String> getDatasourceProperties(final Datasource datasource) {
Map<String, String> returnProps = new HashMap<String, String>();
if (datasource.getProperties() != null) {
for (Property prop : datasource.getProperties().getProperties()) {
returnProps.put(prop.getName(), prop.getValue());
}
}
return returnProps;
}
}