package org.apache.pig.impl.io;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.data.Tuple;
import eu.stratosphere.nephele.configuration.Configuration;
import eu.stratosphere.nephele.configuration.GlobalConfiguration;
import eu.stratosphere.nephele.fs.BlockLocation;
import eu.stratosphere.nephele.fs.FSDataInputStream;
import eu.stratosphere.nephele.fs.FileInputSplit;
import eu.stratosphere.nephele.fs.FileStatus;
import eu.stratosphere.nephele.fs.FileSystem;
import eu.stratosphere.nephele.fs.Path;
import eu.stratosphere.pact.common.contract.FileDataSource;
import eu.stratosphere.pact.common.generic.io.InputFormat;
import eu.stratosphere.pact.common.io.statistics.BaseStatistics;
import eu.stratosphere.pact.common.util.PactConfigConstants;
/**
*
* Copy of FileInputFormat with the only difference that the methods
* readRecord() and nextRecord() have Tuple as targets instead of PactRecord
* @author hinata
*
*/
public abstract class SPigFileInputFormat implements InputFormat<Tuple, FileInputSplit> {
/**
* The log.
*/
private static final Log LOG = LogFactory.getLog(SPigFileInputFormat.class);
/**
* The timeout (in milliseconds) to wait for a filesystem stream to respond.
*/
static final long DEFAULT_OPENING_TIMEOUT;
static {
final long to = GlobalConfiguration.getLong(PactConfigConstants.FS_STREAM_OPENING_TIMEOUT_KEY,
PactConfigConstants.DEFAULT_FS_STREAM_OPENING_TIMEOUT);
if (to < 0) {
LOG.error("Invalid timeout value for filesystem stream opening: " + to + ". Using default value of " +
PactConfigConstants.DEFAULT_FS_STREAM_OPENING_TIMEOUT);
DEFAULT_OPENING_TIMEOUT = PactConfigConstants.DEFAULT_FS_STREAM_OPENING_TIMEOUT;
} else if (to == 0) {
DEFAULT_OPENING_TIMEOUT = Long.MAX_VALUE;
} else {
DEFAULT_OPENING_TIMEOUT = to;
}
}
/**
* The fraction that the last split may be larger than the others.
*/
private static final float MAX_SPLIT_SIZE_DISCREPANCY = 1.1f;
// ------------------------------------- Config Keys ------------------------------------------
/**
* The config parameter which defines the input file path.
*/
public static final String FILE_PARAMETER_KEY = "pact.input.file.path";
/**
* The config parameter which defines the number of desired splits.
*/
private static final String DESIRED_NUMBER_OF_SPLITS_PARAMETER_KEY = "pact.input.file.numsplits";
/**
* The config parameter for the minimal split size.
*/
private static final String MINIMAL_SPLIT_SIZE_PARAMETER_KEY = "pact.input.file.minsplitsize";
/**
* The config parameter for the opening timeout in milliseconds.
*/
public static final String INPUT_STREAM_OPEN_TIMEOUT_KEY = "pact.input.file.timeout";
// --------------------------------------------------------------------------------------------
/**
* The path to the file that contains the input.
*/
protected Path filePath;
/**
* The input stream reading from the input file.
*/
protected FSDataInputStream stream;
/**
* The start of the split that this parallel instance must consume.
*/
protected long splitStart;
/**
* The length of the split that this parallel instance must consume.
*/
protected long splitLength;
/**
* The the minimal split size, set by the configure() method.
*/
protected long minSplitSize;
/**
* The desired number of splits, as set by the configure() method.
*/
protected int numSplits;
/**
* Stream opening timeout.
*/
private long openTimeout;
// --------------------------------------------------------------------------------------------
/**
* Configures the file input format by reading the file path from the configuration.
*
* @see eu.stratosphere.pact.common.generic.io.InputFormat#configure(eu.stratosphere.nephele.configuration.Configuration)
*/
@Override
public void configure(Configuration parameters)
{
// get the file path
final String filePath = parameters.getString(FILE_PARAMETER_KEY, null);
if (filePath == null) {
throw new IllegalArgumentException("Configuration file SPigFileInputFormat does not contain the file path.");
}
try {
this.filePath = new Path(filePath);
}
catch (RuntimeException rex) {
throw new RuntimeException("Could not create a valid URI from the given file path name: " + rex.getMessage());
}
// get the number of splits
this.numSplits = parameters.getInteger(DESIRED_NUMBER_OF_SPLITS_PARAMETER_KEY, -1);
if (this.numSplits == 0 || this.numSplits < -1) {
this.numSplits = -1;
if (LOG.isWarnEnabled())
LOG.warn("Ignoring invalid parameter for number of splits: " + this.numSplits);
}
// get the minimal split size
this.minSplitSize = parameters.getLong(MINIMAL_SPLIT_SIZE_PARAMETER_KEY, 1);
if (this.minSplitSize < 1) {
this.minSplitSize = 1;
if (LOG.isWarnEnabled())
LOG.warn("Ignoring invalid parameter for minimal split size (requires a positive value): " + this.numSplits);
}
this.openTimeout = parameters.getLong(INPUT_STREAM_OPEN_TIMEOUT_KEY, DEFAULT_OPENING_TIMEOUT);
if (this.openTimeout < 0) {
this.openTimeout = DEFAULT_OPENING_TIMEOUT;
if (LOG.isWarnEnabled())
LOG.warn("Ignoring invalid parameter for stream opening timeout (requires a positive value or zero=infinite): " + this.openTimeout);
} else if (this.openTimeout == 0) {
this.openTimeout = Long.MAX_VALUE;
}
}
/* (non-Javadoc)
* @see eu.stratosphere.pact.common.io.InputFormat#getInputSplitType()
*/
@Override
public Class<FileInputSplit> getInputSplitType()
{
return FileInputSplit.class;
}
/**
* Computes the input splits for the file. By default, one file block is one split. If more splits
* are requested than blocks are available, then a split may by a fraction of a block and splits may cross
* block boundaries.
*
* @param The minimum desired number of file splits.
* @return The computed file splits.
* @see eu.stratosphere.pact.common.generic.io.InputFormat#createInputSplits(int)
*/
@Override
public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException
{
// take the desired number of splits into account
minNumSplits = Math.max(minNumSplits, this.numSplits);
final Path path = this.filePath;
final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>(minNumSplits);
// get all the files that are involved in the splits
List<FileStatus> files = new ArrayList<FileStatus>();
long totalLength = 0;
final FileSystem fs = path.getFileSystem();
final FileStatus pathFile = fs.getFileStatus(path);
if (pathFile.isDir()) {
// input is directory. list all contained files
final FileStatus[] dir = fs.listStatus(path);
for (int i = 0; i < dir.length; i++) {
if (!dir[i].isDir()) {
files.add(dir[i]);
totalLength += dir[i].getLen();
}
}
} else {
files.add(pathFile);
totalLength += pathFile.getLen();
}
final long maxSplitSize = (minNumSplits < 1) ? Long.MAX_VALUE : (totalLength / minNumSplits +
(totalLength % minNumSplits == 0 ? 0 : 1));
// now that we have the files, generate the splits
int splitNum = 0;
for (final FileStatus file : files) {
final long len = file.getLen();
final long blockSize = file.getBlockSize();
final long minSplitSize;
if (this.minSplitSize <= blockSize) {
minSplitSize = this.minSplitSize;
}
else {
if (LOG.isWarnEnabled())
LOG.warn("Minimal split size of " + this.minSplitSize + " is larger than the block size of " +
blockSize + ". Decreasing minimal split size to block size.");
minSplitSize = blockSize;
}
final long splitSize = Math.max(minSplitSize, Math.min(maxSplitSize, blockSize));
final long halfSplit = splitSize >>> 1;
final long maxBytesForLastSplit = (long) (splitSize * MAX_SPLIT_SIZE_DISCREPANCY);
if (len > 0) {
// get the block locations and make sure they are in order with respect to their offset
final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len);
Arrays.sort(blocks);
long bytesUnassigned = len;
long position = 0;
int blockIndex = 0;
while (bytesUnassigned > maxBytesForLastSplit) {
// get the block containing the majority of the data
blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex);
// create a new split
FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, splitSize,
blocks[blockIndex].getHosts());
inputSplits.add(fis);
// adjust the positions
position += splitSize;
bytesUnassigned -= splitSize;
}
// assign the last split
if (bytesUnassigned > 0) {
blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex);
final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position,
bytesUnassigned, blocks[blockIndex].getHosts());
inputSplits.add(fis);
}
} else {
// special case with a file of zero bytes size
final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, 0);
String[] hosts;
if (blocks.length > 0) {
hosts = blocks[0].getHosts();
} else {
hosts = new String[0];
}
final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), 0, 0, hosts);
inputSplits.add(fis);
}
}
return inputSplits.toArray(new FileInputSplit[inputSplits.size()]);
}
/**
* Retrieves the index of the <tt>BlockLocation</tt> that contains the part of the file described by the given
* offset.
*
* @param blocks The different blocks of the file. Must be ordered by their offset.
* @param offset The offset of the position in the file.
* @param startIndex The earliest index to look at.
* @return The index of the block containing the given position.
*/
private final int getBlockIndexForPosition(BlockLocation[] blocks, long offset, long halfSplitSize, int startIndex)
{
// go over all indexes after the startIndex
for (int i = startIndex; i < blocks.length; i++) {
long blockStart = blocks[i].getOffset();
long blockEnd = blockStart + blocks[i].getLength();
if (offset >= blockStart && offset < blockEnd) {
// got the block where the split starts
// check if the next block contains more than this one does
if (i < blocks.length - 1 && blockEnd - offset < halfSplitSize) {
return i + 1;
} else {
return i;
}
}
}
throw new IllegalArgumentException("The given offset is not contained in the any block.");
}
// --------------------------------------------------------------------------------------------
/**
* Opens an input stream to the file defined in the input format.
* The stream is positioned at the beginning of the given split.
* <p>
* The stream is actually opened in an asynchronous thread to make sure any interruptions to the thread
* working on the input format do not reach the file system.
*
* @see eu.stratosphere.pact.common.generic.io.InputFormat#open(eu.stratosphere.nephele.template.InputSplit)
*/
@Override
public void open(FileInputSplit split) throws IOException
{
if (!(split instanceof FileInputSplit)) {
throw new IllegalArgumentException("File Input Formats can only be used with FileInputSplits.");
}
final FileInputSplit fileSplit = (FileInputSplit) split;
this.splitStart = fileSplit.getStart();
this.splitLength = fileSplit.getLength();
if (LOG.isDebugEnabled())
LOG.debug("Opening input split " + fileSplit.getPath() + " [" + this.splitStart + "," + this.splitLength + "]");
// open the split in an asynchronous thread
final InputSplitOpenThread isot = new InputSplitOpenThread(fileSplit, this.openTimeout);
isot.start();
try {
this.stream = isot.waitForCompletion();
}
catch (Throwable t) {
throw new IOException("Error opening the Input Split " + fileSplit.getPath() +
" [" + splitStart + "," + splitLength + "]: " + t.getMessage(), t);
}
// get FSDataInputStream
this.stream.seek(this.splitStart);
}
/**
* Closes the file input stream of the input format.
*/
@Override
public void close() throws IOException
{
if (this.stream != null) {
// close input stream
this.stream.close();
}
}
/* (non-Javadoc)
* @see java.lang.Object#toString()
*/
public String toString()
{
return this.filePath == null ?
"File Input (unknown file)" :
"File Input (" + this.filePath.toString() + ')';
}
// ============================================================================================
/**
* Encapsulation of the basic statistics the optimizer obtains about a file. Contained are the size of the file
* and the average bytes of a single record. The statistics also have a time-stamp that records the modification
* time of the file and indicates as such for which time the statistics were valid.
*
* @author Stephan Ewen (stephan.ewen@tu-berlin.de)
*/
public static class FileBaseStatistics implements BaseStatistics
{
protected long fileModTime; // timestamp of the last modification
protected long fileSize; // size of the file(s) in bytes
protected float avgBytesPerRecord; // the average number of bytes for a record
/**
* Creates a new statistics object.
*
* @param fileModTime
* The timestamp of the latest modification of any of the involved files.
* @param fileSize
* The size of the file, in bytes. <code>-1</code>, if unknown.
* @param avgBytesPerRecord
* The average number of byte in a record, or <code>-1.0f</code>, if unknown.
*/
public FileBaseStatistics(long fileModTime, long fileSize, float avgBytesPerRecord) {
this.fileModTime = fileModTime;
this.fileSize = fileSize;
this.avgBytesPerRecord = avgBytesPerRecord;
}
/**
* Gets the timestamp of the last modification.
*
* @return The timestamp of the last modification.
*/
public long getLastModificationTime() {
return fileModTime;
}
/**
* Sets the timestamp of the last modification.
*
* @param modificationTime The timestamp of the last modification
*/
public void setLastModificationTime(long modificationTime)
{
this.fileModTime = modificationTime;
}
/**
* Gets the file size.
*
* @return The fileSize.
* @see eu.stratosphere.pact.common.io.statistics.BaseStatistics#getTotalInputSize()
*/
@Override
public long getTotalInputSize()
{
return this.fileSize;
}
/**
* Sets the file size to the specified value.
*
* @param fileSize the fileSize to set
*/
public void setTotalInputSize(long fileSize)
{
this.fileSize = fileSize;
}
/**
* Gets the estimates number of records in the file, computed as the file size divided by the
* average record width, rounded up.
*
* @return The estimated number of records in the file.
* @see eu.stratosphere.pact.common.io.statistics.BaseStatistics#getNumberOfRecords()
*/
@Override
public long getNumberOfRecords()
{
return (long) Math.ceil(this.fileSize / this.avgBytesPerRecord);
}
/**
* Sets the estimated average number of bytes per record.
*
* @param avgBytesPerRecord the average number of bytes per record
*/
public void setAverageRecordWidth(float avgBytesPerRecord)
{
this.avgBytesPerRecord = avgBytesPerRecord;
}
/**
* Gets the estimated average number of bytes per record.
*
* @return The average number of bytes per record.
* @see eu.stratosphere.pact.common.io.statistics.BaseStatistics#getAverageRecordWidth()
*/
@Override
public float getAverageRecordWidth()
{
return this.avgBytesPerRecord;
}
}
// ============================================================================================
/**
* Obtains a DataInputStream in an thread that is not interrupted.
* This is a necessary hack around the problem that the HDFS client is very sensitive to InterruptedExceptions.
*/
public static class InputSplitOpenThread extends Thread
{
private final FileInputSplit split;
private final long timeout;
private volatile FSDataInputStream fdis;
private volatile Throwable error;
private volatile boolean aborted;
public InputSplitOpenThread(FileInputSplit split, long timeout)
{
super("Transient InputSplit Opener");
setDaemon(true);
this.split = split;
this.timeout = timeout;
}
@Override
public void run()
{
try {
final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
this.fdis = fs.open(this.split.getPath());
// check for canceling and close the stream in that case, because no one will obtain it
if (this.aborted) {
final FSDataInputStream f = this.fdis;
this.fdis = null;
f.close();
}
}
catch (Throwable t) {
this.error = t;
}
}
public FSDataInputStream waitForCompletion() throws Throwable
{
final long start = System.currentTimeMillis();
long remaining = this.timeout;
do {
try {
// wait for the task completion
this.join(remaining);
}
catch (InterruptedException iex) {
// we were canceled, so abort the procedure
abortWait();
throw iex;
}
}
while (this.error == null && this.fdis == null &&
(remaining = this.timeout + start - System.currentTimeMillis()) > 0);
if (this.error != null) {
throw this.error;
}
if (this.fdis != null) {
return this.fdis;
} else {
// double-check that the stream has not been set by now. we don't know here whether
// a) the opener thread recognized the canceling and closed the stream
// b) the flag was set such that the stream did not see it and we have a valid stream
// In any case, close the stream and throw an exception.
abortWait();
final boolean stillAlive = this.isAlive();
final StringBuilder bld = new StringBuilder(256);
for (StackTraceElement e : this.getStackTrace()) {
bld.append("\tat ").append(e.toString()).append('\n');
}
throw new IOException("Input opening request timed out. Opener was " + (stillAlive ? "" : "NOT ") +
" alive. Stack:\n" + bld.toString());
}
}
/**
* Double checked procedure setting the abort flag and closing the stream.
*/
private final void abortWait() {
this.aborted = true;
final FSDataInputStream inStream = this.fdis;
this.fdis = null;
if (inStream != null) {
try {
inStream.close();
} catch (Throwable t) {}
}
}
}
// ============================================================================================
/**
* Creates a configuration builder that can be used to set the input format's parameters to the config in a fluent
* fashion.
*
* @return A config builder for setting parameters.
*/
public static ConfigBuilder configureFileFormat(FileDataSource target) {
return new ConfigBuilder(target.getParameters());
}
/**
* Abstract builder used to set parameters to the input format's configuration in a fluent way.
*/
protected static abstract class AbstractConfigBuilder<T>
{
/**
* The configuration into which the parameters will be written.
*/
protected final Configuration config;
// --------------------------------------------------------------------
/**
* Creates a new builder for the given configuration.
*
* @param targetConfig The configuration into which the parameters will be written.
*/
protected AbstractConfigBuilder(Configuration targetConfig) {
this.config = targetConfig;
}
// --------------------------------------------------------------------
/**
* Sets the desired number of splits for this input format. This value is only a hint. The format
* may create more splits, if there are for example more distributed file blocks (one split is at most
* one block) or less splits, if the minimal split size allows only for fewer splits.
*
* @param numDesiredSplits The desired number of input splits.
* @return The builder itself.
*/
public T desiredSplits(int numDesiredSplits) {
this.config.setInteger(DESIRED_NUMBER_OF_SPLITS_PARAMETER_KEY, numDesiredSplits);
@SuppressWarnings("unchecked")
T ret = (T) this;
return ret;
}
/**
* Sets the minimal size for the splits generated by this input format.
*
* @param minSplitSize The minimal split size, in bytes.
* @return The builder itself.
*/
public T minimumSplitSize(int minSplitSize) {
this.config.setInteger(MINIMAL_SPLIT_SIZE_PARAMETER_KEY, minSplitSize);
@SuppressWarnings("unchecked")
T ret = (T) this;
return ret;
}
/**
* Sets the timeout after which the input format will abort the opening of the input stream,
* if the stream has not responded until then.
*
* @param timeoutInMillies The timeout, in milliseconds, or <code>0</code> for infinite.
* @return The builder itself.
*/
public T openingTimeout(int timeoutInMillies) {
this.config.setLong(INPUT_STREAM_OPEN_TIMEOUT_KEY, timeoutInMillies);
@SuppressWarnings("unchecked")
T ret = (T) this;
return ret;
}
}
/**
* A builder used to set parameters to the input format's configuration in a fluent way.
*/
public static class ConfigBuilder extends AbstractConfigBuilder<ConfigBuilder>
{
/**
* Creates a new builder for the given configuration.
*
* @param targetConfig The configuration into which the parameters will be written.
*/
protected ConfigBuilder(Configuration targetConfig) {
super(targetConfig);
}
}
}