package org.apache.hadoop.conf;
import java.io.IOException;
import java.net.URI;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.json.JSONObject;
public class ClientConfigurationUtil {
private static Log LOG = LogFactory.getLog(ClientConfigurationUtil.class);
private static HashMap<Class<? extends ClientConfiguration>, ClientConfiguration> classCache = new HashMap<Class<? extends ClientConfiguration>, ClientConfiguration>();
// Map to keep track of bad URIs whose client configuration we were not
// able to lookup. We keep this cache around so that we don't repeatedly
// try to query a URI for which the configuration service consistently fails.
private static final HashMap<String, Long> badURIs = new HashMap<String, Long>();
private static final int BAD_URI_EXPIRY = 10 * 60 * 1000; // 10 minutes
/**
* Retrieves the implementation for {@link ClientConfiguration}. The
* expectation is that either this will be specified in the default
* configuration or it will be specified in the configuration of the client
* via command line options and environment variables.
*
* @param conf
* the configuration for the client
* @param uri
* the URI for the filesystem
* @return
*/
public static ClientConfiguration getInstance(URI uri, Configuration conf) {
Class<? extends ClientConfiguration> clazz = conf.getClass("fs." + uri.getScheme()
+ ".client.configuration.impl",
DefaultClientConfiguration.class, ClientConfiguration.class);
ClientConfiguration clientConf = classCache.get(clazz);
if (clientConf == null) {
clientConf = (ClientConfiguration) ReflectionUtils.newInstance(clazz,
null);
classCache.put(clazz, clientConf);
}
return clientConf;
}
/**
* Retrieves a modified version of the configuration from the supplied
* configuration. This is used to override the configuration supplied with the
* client side configuration retried from a different source.
*
* @param uri
* the URI for the FileSystem whose conf is needed
* @param conf
* the default configuration
* @return the modified client side configuration
* @throws IOException
*/
public static Configuration mergeConfiguration(URI uri, Configuration conf)
throws IOException {
try {
Long lastBadAccess = badURIs.get(uri.getHost());
if (lastBadAccess != null) {
if (System.currentTimeMillis() - lastBadAccess < BAD_URI_EXPIRY) {
return conf;
} else {
badURIs.remove(uri.getHost());
}
}
boolean lookupLogical = conf.getBoolean(
"dfs.client.configerator.logical.lookup.enabled", false);
Properties props = new Properties(System.getProperties());
props.setProperty("dfs.client.configerator.logical.lookup.enabled",
lookupLogical + "");
String configDir = conf.get("dfs.client.configerator.dir");
if (configDir != null) {
props.setProperty("dfs.client.configerator.dir", configDir);
}
String json = getInstance(uri, conf).getConfiguration(
uri.getHost(),
conf.getInt("hdfs.retrieve.client_configuration_timeout", 3000),
props);
if (json == null) {
LOG.info("Client configuration lookup disabled/failed. "
+ "Using default configuration");
return conf;
}
Configuration newConf = new Configuration(conf);
JSONObject jsonObj = new JSONObject(json);
Configuration clientConf = new Configuration(jsonObj);
Iterator<Map.Entry<String, String>> it = clientConf.iterator();
while (it.hasNext()) {
Map.Entry<String, String> entry = it.next();
String key = entry.getKey();
String val = entry.getValue();
newConf.set(key, val);
}
newConf.setBoolean("client.configuration.lookup.done", true);
return newConf;
} catch (Throwable t) {
badURIs.put(uri.getHost(), System.currentTimeMillis());
// In case of any error, fallback to the default configuration.
LOG.info("Problem retreiving client side configuration " +
". Using default configuration instead", t);
return conf;
}
}
}