/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
/**
* Class for creating hardlinks.
* Supports Unix/Linux, WinXP/2003/Vista via Cygwin, and Mac OS X.
*
* The HardLink class was formerly a static inner class of FSUtil,
* and the methods provided were blatantly non-thread-safe.
* To enable volume-parallel Update snapshots, we now provide static
* threadsafe methods that allocate new buffer string arrays
* upon each call. We also provide an API to hardlink all files in a
* directory with a single command, which is up to 128 times more
* efficient - and minimizes the impact of the extra buffer creations.
*/
public class HardLink {
public enum OSType {
OS_TYPE_UNIX,
OS_TYPE_WINXP,
OS_TYPE_SOLARIS,
OS_TYPE_MAC
}
public static OSType osType;
private static HardLinkCommandGetter getHardLinkCommand;
public final LinkStats linkStats; //not static
//initialize the command "getters" statically, so can use their
//methods without instantiating the HardLink object
static {
osType = getOSType();
if (osType == OSType.OS_TYPE_WINXP) {
// Windows
getHardLinkCommand = new HardLinkCGWin();
} else {
// Unix
getHardLinkCommand = new HardLinkCGUnix();
//override getLinkCountCommand for the particular Unix variant
//Linux is already set as the default - {"stat","-c%h", null}
if (osType == OSType.OS_TYPE_MAC) {
String[] linkCountCmdTemplate = {"/usr/bin/stat","-f%l", null};
HardLinkCGUnix.setLinkCountCmdTemplate(linkCountCmdTemplate);
} else if (osType == OSType.OS_TYPE_SOLARIS) {
String[] linkCountCmdTemplate = {"ls","-l", null};
HardLinkCGUnix.setLinkCountCmdTemplate(linkCountCmdTemplate);
}
}
}
public HardLink() {
linkStats = new LinkStats();
}
static private OSType getOSType() {
String osName = System.getProperty("os.name");
if (osName.contains("Windows") &&
(osName.contains("XP")
|| osName.contains("2003")
|| osName.contains("Vista")
|| osName.contains("Windows_7")
|| osName.contains("Windows 7")
|| osName.contains("Windows7"))) {
return OSType.OS_TYPE_WINXP;
}
else if (osName.contains("SunOS")
|| osName.contains("Solaris")) {
return OSType.OS_TYPE_SOLARIS;
}
else if (osName.contains("Mac")) {
return OSType.OS_TYPE_MAC;
}
else {
return OSType.OS_TYPE_UNIX;
}
}
/**
* This abstract class bridges the OS-dependent implementations of the
* needed functionality for creating hardlinks and querying link counts.
* The particular implementation class is chosen during
* static initialization phase of the HardLink class.
* The "getter" methods construct shell command strings for various purposes.
*/
private static abstract class HardLinkCommandGetter {
/**
* Get the command string needed to hardlink a bunch of files from
* a single source directory into a target directory. The source directory
* is not specified here, but the command will be executed using the source
* directory as the "current working directory" of the shell invocation.
*
* @param fileBaseNames - array of path-less file names, relative
* to the source directory
* @param linkDir - target directory where the hardlinks will be put
* @return - an array of Strings suitable for use as a single shell command
* with {@link Runtime.exec()}
* @throws IOException - if any of the file or path names misbehave
*/
abstract String[] linkMult(String[] fileBaseNames, File linkDir)
throws IOException;
/**
* Get the command string needed to hardlink a single file
*/
abstract String[] linkOne(File file, File linkName) throws IOException;
/**
* Get the command string to query the hardlink count of a file
*/
abstract String[] linkCount(File file) throws IOException;
/**
* Calculate the total string length of the shell command
* resulting from execution of linkMult, plus the length of the
* source directory name (which will also be provided to the shell)
*
* @param fileDir - source directory, parent of fileBaseNames
* @param fileBaseNames - array of path-less file names, relative
* to the source directory
* @param linkDir - target directory where the hardlinks will be put
* @return - total data length (must not exceed maxAllowedCmdArgLength)
* @throws IOException
*/
abstract int getLinkMultArgLength(
File fileDir, String[] fileBaseNames, File linkDir)
throws IOException;
/**
* Get the maximum allowed string length of a shell command on this OS,
* which is just the documented minimum guaranteed supported command
* length - aprx. 32KB for Unix, and 8KB for Windows.
*/
abstract int getMaxAllowedCmdArgLength();
}
/**
* Implementation of HardLinkCommandGetter class for Unix
*/
static class HardLinkCGUnix extends HardLinkCommandGetter {
private static String[] hardLinkCommand = {"ln", null, null};
private static String[] hardLinkMultPrefix = {"ln"};
private static String[] hardLinkMultSuffix = {null};
private static String[] getLinkCountCommand = {"stat","-c%h", null};
//Unix guarantees at least 32K bytes cmd length.
//Subtract another 64b to allow for Java 'exec' overhead
private static final int maxAllowedCmdArgLength = 32*1024 - 65;
private static synchronized
void setLinkCountCmdTemplate(String[] template) {
//May update this for specific unix variants,
//after static initialization phase
getLinkCountCommand = template;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkOne(java.io.File, java.io.File)
*/
@Override
String[] linkOne(File file, File linkName)
throws IOException {
String[] buf = new String[hardLinkCommand.length];
System.arraycopy(hardLinkCommand, 0, buf, 0, hardLinkCommand.length);
//unix wants argument order: "ln <existing> <new>"
buf[1] = FileUtil.makeShellPath(file, true);
buf[2] = FileUtil.makeShellPath(linkName, true);
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkMult(java.lang.String[], java.io.File)
*/
@Override
String[] linkMult(String[] fileBaseNames, File linkDir)
throws IOException {
String[] buf = new String[fileBaseNames.length
+ hardLinkMultPrefix.length
+ hardLinkMultSuffix.length];
int mark=0;
System.arraycopy(hardLinkMultPrefix, 0, buf, mark,
hardLinkMultPrefix.length);
mark += hardLinkMultPrefix.length;
System.arraycopy(fileBaseNames, 0, buf, mark, fileBaseNames.length);
mark += fileBaseNames.length;
buf[mark] = FileUtil.makeShellPath(linkDir, true);
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkCount(java.io.File)
*/
@Override
String[] linkCount(File file)
throws IOException {
String[] buf = new String[getLinkCountCommand.length];
System.arraycopy(getLinkCountCommand, 0, buf, 0,
getLinkCountCommand.length);
buf[getLinkCountCommand.length - 1] = FileUtil.makeShellPath(file, true);
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getLinkMultArgLength(java.io.File, java.lang.String[], java.io.File)
*/
@Override
int getLinkMultArgLength(File fileDir, String[] fileBaseNames, File linkDir)
throws IOException{
int sum = 0;
for (String x : fileBaseNames) {
// add 1 to account for terminal null or delimiter space
sum += 1 + ((x == null) ? 0 : x.length());
}
sum += 2 + FileUtil.makeShellPath(fileDir, true).length()
+ FileUtil.makeShellPath(linkDir, true).length();
//add the fixed overhead of the hardLinkMult prefix and suffix
sum += 3; //length("ln") + 1
return sum;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getMaxAllowedCmdArgLength()
*/
@Override
int getMaxAllowedCmdArgLength() {
return maxAllowedCmdArgLength;
}
}
/**
* Implementation of HardLinkCommandGetter class for Windows
*
* Note that the linkCount shell command for Windows is actually
* a Cygwin shell command, and depends on ${cygwin}/bin
* being in the Windows PATH environment variable, so
* stat.exe can be found.
*/
static class HardLinkCGWin extends HardLinkCommandGetter {
//The Windows command getter impl class and its member fields are
//package-private ("default") access instead of "private" to assist
//unit testing (sort of) on non-Win servers
static String[] hardLinkCommand = {
"fsutil","hardlink","create", null, null};
static String[] hardLinkMultPrefix = {
"cmd","/q","/c","for", "%f", "in", "("};
static String hardLinkMultDir = "\\%f";
static String[] hardLinkMultSuffix = {
")", "do", "fsutil", "hardlink", "create", null,
"%f", "1>NUL"};
static String[] getLinkCountCommand = {"stat","-c%h", null};
//Windows guarantees only 8K - 1 bytes cmd length.
//Subtract another 64b to allow for Java 'exec' overhead
static final int maxAllowedCmdArgLength = 8*1024 - 65;
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkOne(java.io.File, java.io.File)
*/
@Override
String[] linkOne(File file, File linkName)
throws IOException {
String[] buf = new String[hardLinkCommand.length];
System.arraycopy(hardLinkCommand, 0, buf, 0, hardLinkCommand.length);
//windows wants argument order: "create <new> <existing>"
buf[4] = file.getCanonicalPath();
buf[3] = linkName.getCanonicalPath();
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkMult(java.lang.String[], java.io.File)
*/
@Override
String[] linkMult(String[] fileBaseNames, File linkDir)
throws IOException {
String[] buf = new String[fileBaseNames.length
+ hardLinkMultPrefix.length
+ hardLinkMultSuffix.length];
String td = linkDir.getCanonicalPath() + hardLinkMultDir;
int mark=0;
System.arraycopy(hardLinkMultPrefix, 0, buf, mark,
hardLinkMultPrefix.length);
mark += hardLinkMultPrefix.length;
System.arraycopy(fileBaseNames, 0, buf, mark, fileBaseNames.length);
mark += fileBaseNames.length;
System.arraycopy(hardLinkMultSuffix, 0, buf, mark,
hardLinkMultSuffix.length);
mark += hardLinkMultSuffix.length;
buf[mark - 3] = td;
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkCount(java.io.File)
*/
@Override
String[] linkCount(File file)
throws IOException {
String[] buf = new String[getLinkCountCommand.length];
System.arraycopy(getLinkCountCommand, 0, buf, 0,
getLinkCountCommand.length);
//The linkCount command is actually a Cygwin shell command,
//not a Windows shell command, so we should use "makeShellPath()"
//instead of "getCanonicalPath()". However, that causes another
//shell exec to "cygpath.exe", and "stat.exe" actually can handle
//DOS-style paths (it just prints a couple hundred bytes of warning
//to stderr), so we use the more efficient "getCanonicalPath()".
buf[getLinkCountCommand.length - 1] = file.getCanonicalPath();
return buf;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getLinkMultArgLength(java.io.File, java.lang.String[], java.io.File)
*/
@Override
int getLinkMultArgLength(File fileDir, String[] fileBaseNames, File linkDir)
throws IOException {
int sum = 0;
for (String x : fileBaseNames) {
// add 1 to account for terminal null or delimiter space
sum += 1 + ((x == null) ? 0 : x.length());
}
sum += 2 + fileDir.getCanonicalPath().length() +
linkDir.getCanonicalPath().length();
//add the fixed overhead of the hardLinkMult command
//(prefix, suffix, and Dir suffix)
sum += ("cmd.exe /q /c for %f in ( ) do "
+ "fsutil hardlink create \\%f %f 1>NUL ").length();
return sum;
}
/*
* @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getMaxAllowedCmdArgLength()
*/
@Override
int getMaxAllowedCmdArgLength() {
return maxAllowedCmdArgLength;
}
}
/**
* Calculate the nominal length of all contributors to the total
* commandstring length, including fixed overhead of the OS-dependent
* command. It's protected rather than private, to assist unit testing,
* but real clients are not expected to need it -- see the way
* createHardLinkMult() uses it internally so the user doesn't need to worry
* about it.
*
* @param fileDir - source directory, parent of fileBaseNames
* @param fileBaseNames - array of path-less file names, relative
* to the source directory
* @param linkDir - target directory where the hardlinks will be put
* @return - total data length (must not exceed maxAllowedCmdArgLength)
* @throws IOException
*/
protected static int getLinkMultArgLength(
File fileDir, String[] fileBaseNames, File linkDir)
throws IOException {
return getHardLinkCommand.getLinkMultArgLength(fileDir,
fileBaseNames, linkDir);
}
/**
* Return this private value for use by unit tests.
* Shell commands are not allowed to have a total string length
* exceeding this size.
*/
protected static int getMaxAllowedCmdArgLength() {
return getHardLinkCommand.getMaxAllowedCmdArgLength();
}
/*
* ****************************************************
* Complexity is above. User-visible functionality is below
* ****************************************************
*/
/**
* Creates a hardlink
* @param file - existing source file
* @param linkName - desired target link file
*/
public static void createHardLink(File file, File linkName)
throws IOException {
if (file == null) {
throw new IOException(
"invalid arguments to createHardLink: source file is null");
}
if (linkName == null) {
throw new IOException(
"invalid arguments to createHardLink: link name is null");
}
// construct and execute shell command
String[] hardLinkCommand = getHardLinkCommand.linkOne(file, linkName);
Process process = Runtime.getRuntime().exec(hardLinkCommand);
try {
if (process.waitFor() != 0) {
String errMsg = new BufferedReader(new InputStreamReader(
process.getInputStream())).readLine();
if (errMsg == null) errMsg = "";
String inpMsg = new BufferedReader(new InputStreamReader(
process.getErrorStream())).readLine();
if (inpMsg == null) inpMsg = "";
throw new IOException(errMsg + inpMsg);
}
} catch (InterruptedException e) {
throw new IOException(e);
} finally {
process.destroy();
}
}
/**
* Creates hardlinks from multiple existing files within one parent
* directory, into one target directory.
* @param parentDir - directory containing source files
* @param fileBaseNames - list of path-less file names, as returned by
* parentDir.list()
* @param linkDir - where the hardlinks should be put. It must already exist.
*
* If the list of files is too long (overflows maxAllowedCmdArgLength),
* we will automatically split it into multiple invocations of the
* underlying method.
*/
public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
File linkDir) throws IOException {
//This is the public method all non-test clients are expected to use.
//Normal case - allow up to maxAllowedCmdArgLength characters in the cmd
createHardLinkMult(parentDir, fileBaseNames, linkDir,
getHardLinkCommand.getMaxAllowedCmdArgLength());
}
/*
* Implements {@link createHardLinkMult} with added variable "maxLength",
* to ease unit testing of the auto-splitting feature for long lists.
* Likewise why it returns "callCount", the number of sub-arrays that
* the file list had to be split into.
* Non-test clients are expected to call the public method instead.
*/
protected static int createHardLinkMult(File parentDir,
String[] fileBaseNames, File linkDir, int maxLength)
throws IOException {
if (parentDir == null) {
throw new IOException(
"invalid arguments to createHardLinkMult: parent directory is null");
}
if (linkDir == null) {
throw new IOException(
"invalid arguments to createHardLinkMult: link directory is null");
}
if (fileBaseNames == null) {
throw new IOException(
"invalid arguments to createHardLinkMult: "
+ "filename list can be empty but not null");
}
if (fileBaseNames.length == 0) {
//the OS cmds can't handle empty list of filenames,
//but it's legal, so just return.
return 0;
}
if (!linkDir.exists()) {
throw new FileNotFoundException(linkDir + " not found.");
}
//if the list is too long, split into multiple invocations
int callCount = 0;
if (getLinkMultArgLength(parentDir, fileBaseNames, linkDir) > maxLength
&& fileBaseNames.length > 1) {
String[] list1 = Arrays.copyOf(fileBaseNames, fileBaseNames.length/2);
callCount += createHardLinkMult(parentDir, list1, linkDir, maxLength);
String[] list2 = Arrays.copyOfRange(fileBaseNames, fileBaseNames.length/2,
fileBaseNames.length);
callCount += createHardLinkMult(parentDir, list2, linkDir, maxLength);
return callCount;
} else {
callCount = 1;
}
// construct and execute shell command
String[] hardLinkCommand = getHardLinkCommand.linkMult(fileBaseNames,
linkDir);
Process process = Runtime.getRuntime().exec(hardLinkCommand, null,
parentDir);
try {
if (process.waitFor() != 0) {
String errMsg = new BufferedReader(new InputStreamReader(
process.getInputStream())).readLine();
if (errMsg == null) errMsg = "";
String inpMsg = new BufferedReader(new InputStreamReader(
process.getErrorStream())).readLine();
if (inpMsg == null) inpMsg = "";
throw new IOException(errMsg + inpMsg);
}
} catch (InterruptedException e) {
throw new IOException(e);
} finally {
process.destroy();
}
return callCount;
}
/**
* Retrieves the number of links to the specified file.
*/
public static int getLinkCount(File fileName) throws IOException {
if (fileName == null) {
throw new IOException(
"invalid argument to getLinkCount: file name is null");
}
if (!fileName.exists()) {
throw new FileNotFoundException(fileName + " not found.");
}
// construct and execute shell command
String[] cmd = getHardLinkCommand.linkCount(fileName);
String inpMsg = null;
String errMsg = null;
int exitValue = -1;
BufferedReader in = null;
BufferedReader err = null;
Process process = Runtime.getRuntime().exec(cmd);
try {
exitValue = process.waitFor();
in = new BufferedReader(new InputStreamReader(
process.getInputStream()));
inpMsg = in.readLine();
err = new BufferedReader(new InputStreamReader(
process.getErrorStream()));
errMsg = err.readLine();
if (inpMsg == null || exitValue != 0) {
throw createIOException(fileName, inpMsg, errMsg, exitValue, null);
}
if (osType == OSType.OS_TYPE_SOLARIS) {
String[] result = inpMsg.split("\\s+");
return Integer.parseInt(result[1]);
} else {
return Integer.parseInt(inpMsg);
}
} catch (NumberFormatException e) {
throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
} catch (InterruptedException e) {
throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
} finally {
process.destroy();
if (in != null) in.close();
if (err != null) err.close();
}
}
/* Create an IOException for failing to get link count. */
private static IOException createIOException(File f, String message,
String error, int exitvalue, Exception cause) {
final String winErrMsg = "; Windows errors in getLinkCount are often due "
+ "to Cygwin misconfiguration";
final String s = "Failed to get link count on file " + f
+ ": message=" + message
+ "; error=" + error
+ ((osType == OSType.OS_TYPE_WINXP) ? winErrMsg : "")
+ "; exit value=" + exitvalue;
return (cause == null) ? new IOException(s) : new IOException(s, cause);
}
/**
* HardLink statistics counters and methods.
* Not multi-thread safe, obviously.
* Init is called during HardLink instantiation, above.
*
* These are intended for use by knowledgeable clients, not internally,
* because many of the internal methods are static and can't update these
* per-instance counters.
*/
public static class LinkStats {
public int countDirs = 0;
public int countSingleLinks = 0;
public int countMultLinks = 0;
public int countFilesMultLinks = 0;
public int countEmptyDirs = 0;
public int countPhysicalFileCopies = 0;
public void clear() {
countDirs = 0;
countSingleLinks = 0;
countMultLinks = 0;
countFilesMultLinks = 0;
countEmptyDirs = 0;
countPhysicalFileCopies = 0;
}
public String report() {
return "HardLinkStats: " + countDirs + " Directories, including "
+ countEmptyDirs + " Empty Directories, "
+ countSingleLinks
+ " single Link operations, " + countMultLinks
+ " multi-Link operations, linking " + countFilesMultLinks
+ " files, total " + (countSingleLinks + countFilesMultLinks)
+ " linkable files. Also physically copied "
+ countPhysicalFileCopies + " other files.";
}
}
}