/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.security;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
/**
* A simple shell-based implementation of {@link IdMappingServiceProvider}
* Map id to user name or group name. It does update every 15 minutes. Only a
* single instance of this class is expected to be on the server.
*
* The maps are incrementally updated as described below:
* 1. Initialize the maps as empty.
* 2. Incrementally update the maps
* - When ShellBasedIdMapping is requested for user or group name given
* an ID, or for ID given a user or group name, do look up in the map
* first, if it doesn't exist, find the corresponding entry with shell
* command, and insert the entry to the maps.
* - When group ID is requested for a given group name, and if the
* group name is numerical, the full group map is loaded. Because we
* don't have a good way to find the entry for a numerical group name,
* loading the full map helps to get in all entries.
* 3. Periodically refresh the maps for both user and group, e.g,
* do step 1.
* Note: for testing purpose, step 1 may initial the maps with full mapping
* when using constructor
* {@link ShellBasedIdMapping#ShellBasedIdMapping(Configuration, boolean)}.
*/
public class ShellBasedIdMapping implements IdMappingServiceProvider {
private static final Log LOG =
LogFactory.getLog(ShellBasedIdMapping.class);
private final static String OS = System.getProperty("os.name");
/** Shell commands to get users and groups */
static final String GET_ALL_USERS_CMD = "getent passwd | cut -d: -f1,3";
static final String GET_ALL_GROUPS_CMD = "getent group | cut -d: -f1,3";
static final String MAC_GET_ALL_USERS_CMD = "dscl . -list /Users UniqueID";
static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID";
private final File staticMappingFile;
private StaticMapping staticMapping = null;
// Last time the static map was modified, measured time difference in
// milliseconds since midnight, January 1, 1970 UTC
private long lastModificationTimeStaticMap = 0;
private boolean constructFullMapAtInit = false;
// Used for parsing the static mapping file.
private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$");
private static final Pattern COMMENT_LINE = Pattern.compile("^\\s*#.*$");
private static final Pattern MAPPING_LINE =
Pattern.compile("^(uid|gid)\\s+(\\d+)\\s+(\\d+)\\s*(#.*)?$");
final private long timeout;
// Maps for id to name map. Guarded by this object monitor lock
private BiMap<Integer, String> uidNameMap = HashBiMap.create();
private BiMap<Integer, String> gidNameMap = HashBiMap.create();
private long lastUpdateTime = 0; // Last time maps were updated
/*
* Constructor
* @param conf the configuration
* @param constructFullMapAtInit initialize the maps with full mapping when
* true, otherwise initialize the maps to empty. This parameter is
* intended for testing only, its default is false.
*/
@VisibleForTesting
public ShellBasedIdMapping(Configuration conf,
boolean constructFullMapAtInit) throws IOException {
this.constructFullMapAtInit = constructFullMapAtInit;
long updateTime = conf.getLong(
IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY,
IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT);
// Minimal interval is 1 minute
if (updateTime < IdMappingConstant.USERGROUPID_UPDATE_MILLIS_MIN) {
LOG.info("User configured user account update time is less"
+ " than 1 minute. Use 1 minute instead.");
timeout = IdMappingConstant.USERGROUPID_UPDATE_MILLIS_MIN;
} else {
timeout = updateTime;
}
String staticFilePath =
conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
staticMappingFile = new File(staticFilePath);
updateStaticMapping();
updateMaps();
}
/*
* Constructor
* initialize user and group maps to empty
* @param conf the configuration
*/
public ShellBasedIdMapping(Configuration conf) throws IOException {
this(conf, false);
}
@VisibleForTesting
public long getTimeout() {
return timeout;
}
@VisibleForTesting
public BiMap<Integer, String> getUidNameMap() {
return uidNameMap;
}
@VisibleForTesting
public BiMap<Integer, String> getGidNameMap() {
return gidNameMap;
}
@VisibleForTesting
synchronized public void clearNameMaps() {
uidNameMap.clear();
gidNameMap.clear();
lastUpdateTime = Time.monotonicNow();
}
synchronized private boolean isExpired() {
return Time.monotonicNow() - lastUpdateTime > timeout;
}
// If can't update the maps, will keep using the old ones
private void checkAndUpdateMaps() {
if (isExpired()) {
LOG.info("Update cache now");
try {
updateMaps();
} catch (IOException e) {
LOG.error("Can't update the maps. Will use the old ones,"
+ " which can potentially cause problem.", e);
}
}
}
private static final String DUPLICATE_NAME_ID_DEBUG_INFO =
"NFS gateway could have problem starting with duplicate name or id on the host system.\n"
+ "This is because HDFS (non-kerberos cluster) uses name as the only way to identify a user or group.\n"
+ "The host system with duplicated user/group name or id might work fine most of the time by itself.\n"
+ "However when NFS gateway talks to HDFS, HDFS accepts only user and group name.\n"
+ "Therefore, same name means the same user or same group. To find the duplicated names/ids, one can do:\n"
+ "<getent passwd | cut -d: -f1,3> and <getent group | cut -d: -f1,3> on Linux systems,\n"
+ "<dscl . -list /Users UniqueID> and <dscl . -list /Groups PrimaryGroupID> on MacOS.";
private static void reportDuplicateEntry(final String header,
final Integer key, final String value,
final Integer ekey, final String evalue) {
LOG.warn("\n" + header + String.format(
"new entry (%d, %s), existing entry: (%d, %s).%n%s%n%s",
key, value, ekey, evalue,
"The new entry is to be ignored for the following reason.",
DUPLICATE_NAME_ID_DEBUG_INFO));
}
/**
* uid and gid are defined as uint32 in linux. Some systems create
* (intended or unintended) <nfsnobody, 4294967294> kind of <name,Id>
* mapping, where 4294967294 is 2**32-2 as unsigned int32. As an example,
* https://bugzilla.redhat.com/show_bug.cgi?id=511876.
* Because user or group id are treated as Integer (signed integer or int32)
* here, the number 4294967294 is out of range. The solution is to convert
* uint32 to int32, so to map the out-of-range ID to the negative side of
* Integer, e.g. 4294967294 maps to -2 and 4294967295 maps to -1.
*/
private static Integer parseId(final String idStr) {
long longVal = Long.parseLong(idStr);
return Integer.valueOf((int)longVal);
}
/**
* Get the list of users or groups returned by the specified command,
* and save them in the corresponding map.
* @throws IOException
*/
@VisibleForTesting
public static boolean updateMapInternal(BiMap<Integer, String> map,
String mapName, String command, String regex,
Map<Integer, Integer> staticMapping) throws IOException {
boolean updated = false;
BufferedReader br = null;
try {
Process process = Runtime.getRuntime().exec(
new String[] { "bash", "-c", command });
br = new BufferedReader(
new InputStreamReader(process.getInputStream(),
Charset.defaultCharset()));
String line = null;
while ((line = br.readLine()) != null) {
String[] nameId = line.split(regex);
if ((nameId == null) || (nameId.length != 2)) {
throw new IOException("Can't parse " + mapName + " list entry:" + line);
}
LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]);
// HDFS can't differentiate duplicate names with simple authentication
final Integer key = staticMapping.get(parseId(nameId[1]));
final String value = nameId[0];
if (map.containsKey(key)) {
final String prevValue = map.get(key);
if (value.equals(prevValue)) {
// silently ignore equivalent entries
continue;
}
reportDuplicateEntry(
"Got multiple names associated with the same id: ",
key, value, key, prevValue);
continue;
}
if (map.containsValue(value)) {
final Integer prevKey = map.inverse().get(value);
reportDuplicateEntry(
"Got multiple ids associated with the same name: ",
key, value, prevKey, value);
continue;
}
map.put(key, value);
updated = true;
}
LOG.debug("Updated " + mapName + " map size: " + map.size());
} catch (IOException e) {
LOG.error("Can't update " + mapName + " map");
throw e;
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e1) {
LOG.error("Can't close BufferedReader of command result", e1);
}
}
}
return updated;
}
private boolean checkSupportedPlatform() {
if (!OS.startsWith("Linux") && !OS.startsWith("Mac")) {
LOG.error("Platform is not supported:" + OS
+ ". Can't update user map and group map and"
+ " 'nobody' will be used for any user and group.");
return false;
}
return true;
}
private static boolean isInteger(final String s) {
try {
Integer.parseInt(s);
} catch(NumberFormatException e) {
return false;
}
// only got here if we didn't return false
return true;
}
private synchronized void updateStaticMapping() throws IOException {
final boolean init = (staticMapping == null);
//
// if the static mapping file
// - was modified after last update, load the map again;
// - did not exist but was added since last update, load the map;
// - existed before but deleted since last update, clear the map
//
if (staticMappingFile.exists()) {
// check modification time, reload the file if the last modification
// time changed since prior load.
long lmTime = staticMappingFile.lastModified();
if (lmTime != lastModificationTimeStaticMap) {
LOG.info(init? "Using " : "Reloading " + "'" + staticMappingFile
+ "' for static UID/GID mapping...");
lastModificationTimeStaticMap = lmTime;
staticMapping = parseStaticMap(staticMappingFile);
}
} else {
if (init) {
staticMapping = new StaticMapping(new HashMap<Integer, Integer>(),
new HashMap<Integer, Integer>());
}
if (lastModificationTimeStaticMap != 0 || init) {
// print the following log at initialization or when the static
// mapping file was deleted after prior load
LOG.info("Not doing static UID/GID mapping because '"
+ staticMappingFile + "' does not exist.");
}
lastModificationTimeStaticMap = 0;
staticMapping.clear();
}
}
/*
* Refresh static map, and reset the other maps to empty.
* For testing code, a full map may be re-constructed here when the object
* was created with constructFullMapAtInit being set to true.
*/
synchronized public void updateMaps() throws IOException {
if (!checkSupportedPlatform()) {
return;
}
if (constructFullMapAtInit) {
loadFullMaps();
// set constructFullMapAtInit to false to allow testing code to
// do incremental update to maps after initial construction
constructFullMapAtInit = false;
} else {
updateStaticMapping();
clearNameMaps();
}
}
synchronized private void loadFullUserMap() throws IOException {
BiMap<Integer, String> uMap = HashBiMap.create();
if (OS.startsWith("Mac")) {
updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
staticMapping.uidMapping);
} else {
updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":",
staticMapping.uidMapping);
}
uidNameMap = uMap;
lastUpdateTime = Time.monotonicNow();
}
synchronized private void loadFullGroupMap() throws IOException {
BiMap<Integer, String> gMap = HashBiMap.create();
if (OS.startsWith("Mac")) {
updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
staticMapping.gidMapping);
} else {
updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":",
staticMapping.gidMapping);
}
gidNameMap = gMap;
lastUpdateTime = Time.monotonicNow();
}
synchronized private void loadFullMaps() throws IOException {
loadFullUserMap();
loadFullGroupMap();
}
// search for id with given name, return "<name>:<id>"
// return
// getent group <name> | cut -d: -f1,3
// OR
// id -u <name> | awk '{print "<name>:"$1 }'
//
private String getName2IdCmdLinux(final String name, final boolean isGrp) {
String cmd;
if (isGrp) {
cmd = "getent group " + name + " | cut -d: -f1,3";
} else {
cmd = "id -u " + name + " | awk '{print \"" + name + ":\"$1 }'";
}
return cmd;
}
// search for name with given id, return "<name>:<id>"
private String getId2NameCmdLinux(final int id, final boolean isGrp) {
String cmd = "getent ";
cmd += isGrp? "group " : "passwd ";
cmd += String.valueOf(id) + " | cut -d: -f1,3";
return cmd;
}
// "dscl . -read /Users/<name> | grep UniqueID" returns "UniqueId: <id>",
// "dscl . -read /Groups/<name> | grep PrimaryGroupID" returns "PrimaryGoupID: <id>"
// The following method returns a command that uses awk to process the result,
// of these commands, and returns "<name> <id>", to simulate one entry returned by
// MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD.
// Specificially, this method returns:
// id -u <name> | awk '{print "<name>:"$1 }'
// OR
// dscl . -read /Groups/<name> | grep PrimaryGroupID | awk '($1 == "PrimaryGroupID:") { print "<name> " $2 }'
//
private String getName2IdCmdMac(final String name, final boolean isGrp) {
String cmd;
if (isGrp) {
cmd = "dscl . -read /Groups/" + name;
cmd += " | grep PrimaryGroupID | awk '($1 == \"PrimaryGroupID:\") ";
cmd += "{ print \"" + name + " \" $2 }'";
} else {
cmd = "id -u " + name + " | awk '{print \"" + name + " \"$1 }'";
}
return cmd;
}
// "dscl . -search /Users UniqueID <id>" returns
// <name> UniqueID = (
// <id>
// )
// "dscl . -search /Groups PrimaryGroupID <id>" returns
// <name> PrimaryGroupID = (
// <id>
// )
// The following method returns a command that uses sed to process the
// the result and returns "<name> <id>" to simulate one entry returned
// by MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD.
// For certain negative id case like nfsnobody, the <id> is quoted as
// "<id>", added one sed section to remove the quote.
// Specifically, the method returns:
// dscl . -search /Users UniqueID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/UniqueID =//g' | sed 's/)//g' | sed 's/\"//g'
// OR
// dscl . -search /Groups PrimaryGroupID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/PrimaryGroupID =//g' | sed 's/)//g' | sed 's/\"//g'
//
private String getId2NameCmdMac(final int id, final boolean isGrp) {
String cmd = "dscl . -search /";
cmd += isGrp? "Groups PrimaryGroupID " : "Users UniqueID ";
cmd += String.valueOf(id);
cmd += " | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/";
cmd += isGrp? "PrimaryGroupID" : "UniqueID";
cmd += " = (//g' | sed 's/)//g' | sed 's/\\\"//g'";
return cmd;
}
synchronized private void updateMapIncr(final String name,
final boolean isGrp) throws IOException {
if (!checkSupportedPlatform()) {
return;
}
if (isInteger(name) && isGrp) {
loadFullGroupMap();
return;
}
boolean updated = false;
updateStaticMapping();
if (OS.startsWith("Linux")) {
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getName2IdCmdLinux(name, true), ":",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getName2IdCmdLinux(name, false), ":",
staticMapping.uidMapping);
}
} else {
// Mac
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getName2IdCmdMac(name, true), "\\s+",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getName2IdCmdMac(name, false), "\\s+",
staticMapping.uidMapping);
}
}
if (updated) {
lastUpdateTime = Time.monotonicNow();
}
}
synchronized private void updateMapIncr(final int id,
final boolean isGrp) throws IOException {
if (!checkSupportedPlatform()) {
return;
}
boolean updated = false;
updateStaticMapping();
if (OS.startsWith("Linux")) {
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getId2NameCmdLinux(id, true), ":",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getId2NameCmdLinux(id, false), ":",
staticMapping.uidMapping);
}
} else {
// Mac
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getId2NameCmdMac(id, true), "\\s+",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getId2NameCmdMac(id, false), "\\s+",
staticMapping.uidMapping);
}
}
if (updated) {
lastUpdateTime = Time.monotonicNow();
}
}
@SuppressWarnings("serial")
static final class PassThroughMap<K> extends HashMap<K, K> {
public PassThroughMap() {
this(new HashMap<K, K>());
}
public PassThroughMap(Map<K, K> mapping) {
super();
for (Map.Entry<K, K> entry : mapping.entrySet()) {
super.put(entry.getKey(), entry.getValue());
}
}
@SuppressWarnings("unchecked")
@Override
public K get(Object key) {
if (super.containsKey(key)) {
return super.get(key);
} else {
return (K) key;
}
}
}
@VisibleForTesting
static final class StaticMapping {
final Map<Integer, Integer> uidMapping;
final Map<Integer, Integer> gidMapping;
public StaticMapping(Map<Integer, Integer> uidMapping,
Map<Integer, Integer> gidMapping) {
this.uidMapping = new PassThroughMap<Integer>(uidMapping);
this.gidMapping = new PassThroughMap<Integer>(gidMapping);
}
public void clear() {
uidMapping.clear();
gidMapping.clear();
}
public boolean isNonEmpty() {
return uidMapping.size() > 0 || gidMapping.size() > 0;
}
}
static StaticMapping parseStaticMap(File staticMapFile)
throws IOException {
Map<Integer, Integer> uidMapping = new HashMap<Integer, Integer>();
Map<Integer, Integer> gidMapping = new HashMap<Integer, Integer>();
BufferedReader in = new BufferedReader(new InputStreamReader(
new FileInputStream(staticMapFile), Charsets.UTF_8));
try {
String line = null;
while ((line = in.readLine()) != null) {
// Skip entirely empty and comment lines.
if (EMPTY_LINE.matcher(line).matches() ||
COMMENT_LINE.matcher(line).matches()) {
continue;
}
Matcher lineMatcher = MAPPING_LINE.matcher(line);
if (!lineMatcher.matches()) {
LOG.warn("Could not parse line '" + line + "'. Lines should be of " +
"the form '[uid|gid] [remote id] [local id]'. Blank lines and " +
"everything following a '#' on a line will be ignored.");
continue;
}
// We know the line is fine to parse without error checking like this
// since it matched the regex above.
String firstComponent = lineMatcher.group(1);
Integer remoteId = parseId(lineMatcher.group(2));
Integer localId = parseId(lineMatcher.group(3));
if (firstComponent.equals("uid")) {
uidMapping.put(localId, remoteId);
} else {
gidMapping.put(localId, remoteId);
}
}
} finally {
in.close();
}
return new StaticMapping(uidMapping, gidMapping);
}
synchronized public int getUid(String user) throws IOException {
checkAndUpdateMaps();
Integer id = uidNameMap.inverse().get(user);
if (id == null) {
updateMapIncr(user, false);
id = uidNameMap.inverse().get(user);
if (id == null) {
throw new IOException("User just deleted?:" + user);
}
}
return id.intValue();
}
synchronized public int getGid(String group) throws IOException {
checkAndUpdateMaps();
Integer id = gidNameMap.inverse().get(group);
if (id == null) {
updateMapIncr(group, true);
id = gidNameMap.inverse().get(group);
if (id == null) {
throw new IOException("No such group:" + group);
}
}
return id.intValue();
}
synchronized public String getUserName(int uid, String unknown) {
checkAndUpdateMaps();
String uname = uidNameMap.get(uid);
if (uname == null) {
try {
updateMapIncr(uid, false);
} catch (Exception e) {
}
uname = uidNameMap.get(uid);
if (uname == null) {
LOG.warn("Can't find user name for uid " + uid
+ ". Use default user name " + unknown);
uname = unknown;
}
}
return uname;
}
synchronized public String getGroupName(int gid, String unknown) {
checkAndUpdateMaps();
String gname = gidNameMap.get(gid);
if (gname == null) {
try {
updateMapIncr(gid, true);
} catch (Exception e) {
}
gname = gidNameMap.get(gid);
if (gname == null) {
LOG.warn("Can't find group name for gid " + gid
+ ". Use default group name " + unknown);
gname = unknown;
}
}
return gname;
}
// When can't map user, return user name's string hashcode
public int getUidAllowingUnknown(String user) {
checkAndUpdateMaps();
int uid;
try {
uid = getUid(user);
} catch (IOException e) {
uid = user.hashCode();
LOG.info("Can't map user " + user + ". Use its string hashcode:" + uid);
}
return uid;
}
// When can't map group, return group name's string hashcode
public int getGidAllowingUnknown(String group) {
checkAndUpdateMaps();
int gid;
try {
gid = getGid(group);
} catch (IOException e) {
gid = group.hashCode();
LOG.info("Can't map group " + group + ". Use its string hashcode:" + gid);
}
return gid;
}
}