/*
* Copyright (C) 2008-2009, Google Inc.
* Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.util;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ProgressMonitor;
/**
* A fully buffered output stream.
* <p>
* Subclasses determine the behavior when the in-memory buffer capacity has been
* exceeded and additional bytes are still being received for output.
*/
public abstract class TemporaryBuffer extends OutputStream {
/** Default limit for in-core storage. */
protected static final int DEFAULT_IN_CORE_LIMIT = 1024 * 1024;
/** Chain of data, if we are still completely in-core; otherwise null. */
ArrayList<Block> blocks;
/**
* Maximum number of bytes we will permit storing in memory.
* <p>
* When this limit is reached the data will be shifted to a file on disk,
* preventing the JVM heap from growing out of control.
*/
private int inCoreLimit;
/** Initial size of block list. */
private int initialBlocks;
/** If {@link #inCoreLimit} has been reached, remainder goes here. */
private OutputStream overflow;
/**
* Create a new empty temporary buffer.
*
* @param limit
* maximum number of bytes to store in memory before entering the
* overflow output path; also used as the estimated size.
*/
protected TemporaryBuffer(final int limit) {
this(limit, limit);
}
/**
* Create a new empty temporary buffer.
*
* @param estimatedSize
* estimated size of storage used, to size the initial list of
* block pointers.
* @param limit
* maximum number of bytes to store in memory before entering the
* overflow output path.
* @since 4.0
*/
protected TemporaryBuffer(final int estimatedSize, final int limit) {
if (estimatedSize > limit)
throw new IllegalArgumentException();
this.inCoreLimit = limit;
this.initialBlocks = (estimatedSize - 1) / Block.SZ + 1;
reset();
}
@Override
public void write(final int b) throws IOException {
if (overflow != null) {
overflow.write(b);
return;
}
Block s = last();
if (s.isFull()) {
if (reachedInCoreLimit()) {
overflow.write(b);
return;
}
s = new Block();
blocks.add(s);
}
s.buffer[s.count++] = (byte) b;
}
@Override
public void write(final byte[] b, int off, int len) throws IOException {
if (overflow == null) {
while (len > 0) {
Block s = last();
if (s.isFull()) {
if (reachedInCoreLimit())
break;
s = new Block();
blocks.add(s);
}
final int n = Math.min(s.buffer.length - s.count, len);
System.arraycopy(b, off, s.buffer, s.count, n);
s.count += n;
len -= n;
off += n;
}
}
if (len > 0)
overflow.write(b, off, len);
}
/**
* Dumps the entire buffer into the overflow stream, and flushes it.
*
* @throws IOException
* the overflow stream cannot be started, or the buffer contents
* cannot be written to it, or it failed to flush.
*/
protected void doFlush() throws IOException {
if (overflow == null)
switchToOverflow();
overflow.flush();
}
/**
* Copy all bytes remaining on the input stream into this buffer.
*
* @param in
* the stream to read from, until EOF is reached.
* @throws IOException
* an error occurred reading from the input stream, or while
* writing to a local temporary file.
*/
public void copy(final InputStream in) throws IOException {
if (blocks != null) {
for (;;) {
Block s = last();
if (s.isFull()) {
if (reachedInCoreLimit())
break;
s = new Block();
blocks.add(s);
}
int n = in.read(s.buffer, s.count, s.buffer.length - s.count);
if (n < 1)
return;
s.count += n;
}
}
final byte[] tmp = new byte[Block.SZ];
int n;
while ((n = in.read(tmp)) > 0)
overflow.write(tmp, 0, n);
}
/**
* Obtain the length (in bytes) of the buffer.
* <p>
* The length is only accurate after {@link #close()} has been invoked.
*
* @return total length of the buffer, in bytes.
*/
public long length() {
return inCoreLength();
}
private long inCoreLength() {
final Block last = last();
return ((long) blocks.size() - 1) * Block.SZ + last.count;
}
/**
* Convert this buffer's contents into a contiguous byte array.
* <p>
* The buffer is only complete after {@link #close()} has been invoked.
*
* @return the complete byte array; length matches {@link #length()}.
* @throws IOException
* an error occurred reading from a local temporary file
* @throws OutOfMemoryError
* the buffer cannot fit in memory
*/
public byte[] toByteArray() throws IOException {
final long len = length();
if (Integer.MAX_VALUE < len)
throw new OutOfMemoryError(JGitText.get().lengthExceedsMaximumArraySize);
final byte[] out = new byte[(int) len];
int outPtr = 0;
for (final Block b : blocks) {
System.arraycopy(b.buffer, 0, out, outPtr, b.count);
outPtr += b.count;
}
return out;
}
/**
* Convert this buffer's contents into a contiguous byte array. If this size
* of the buffer exceeds the limit only return the first {@code limit} bytes
* <p>
* The buffer is only complete after {@link #close()} has been invoked.
*
* @param limit
* the maximum number of bytes to be returned
*
* @return the byte array limited to {@code limit} bytes.
* @throws IOException
* an error occurred reading from a local temporary file
* @throws OutOfMemoryError
* the buffer cannot fit in memory
*
* @since 4.2
*/
public byte[] toByteArray(int limit) throws IOException {
final long len = Math.min(length(), limit);
if (Integer.MAX_VALUE < len)
throw new OutOfMemoryError(
JGitText.get().lengthExceedsMaximumArraySize);
final byte[] out = new byte[(int) len];
int outPtr = 0;
for (final Block b : blocks) {
System.arraycopy(b.buffer, 0, out, outPtr, b.count);
outPtr += b.count;
}
return out;
}
/**
* Send this buffer to an output stream.
* <p>
* This method may only be invoked after {@link #close()} has completed
* normally, to ensure all data is completely transferred.
*
* @param os
* stream to send this buffer's complete content to.
* @param pm
* if not null progress updates are sent here. Caller should
* initialize the task and the number of work units to <code>
* {@link #length()}/1024</code>.
* @throws IOException
* an error occurred reading from a temporary file on the local
* system, or writing to the output stream.
*/
public void writeTo(final OutputStream os, ProgressMonitor pm)
throws IOException {
if (pm == null)
pm = NullProgressMonitor.INSTANCE;
for (final Block b : blocks) {
os.write(b.buffer, 0, b.count);
pm.update(b.count / 1024);
}
}
/**
* Open an input stream to read from the buffered data.
* <p>
* This method may only be invoked after {@link #close()} has completed
* normally, to ensure all data is completely transferred.
*
* @return a stream to read from the buffer. The caller must close the
* stream when it is no longer useful.
* @throws IOException
* an error occurred opening the temporary file.
*/
public InputStream openInputStream() throws IOException {
return new BlockInputStream();
}
/** Reset this buffer for reuse, purging all buffered content. */
public void reset() {
if (overflow != null) {
destroy();
}
if (blocks != null)
blocks.clear();
else
blocks = new ArrayList<Block>(initialBlocks);
blocks.add(new Block(Math.min(inCoreLimit, Block.SZ)));
}
/**
* Open the overflow output stream, so the remaining output can be stored.
*
* @return the output stream to receive the buffered content, followed by
* the remaining output.
* @throws IOException
* the buffer cannot create the overflow stream.
*/
protected abstract OutputStream overflow() throws IOException;
private Block last() {
return blocks.get(blocks.size() - 1);
}
private boolean reachedInCoreLimit() throws IOException {
if (inCoreLength() < inCoreLimit)
return false;
switchToOverflow();
return true;
}
private void switchToOverflow() throws IOException {
overflow = overflow();
final Block last = blocks.remove(blocks.size() - 1);
for (final Block b : blocks)
overflow.write(b.buffer, 0, b.count);
blocks = null;
overflow = new BufferedOutputStream(overflow, Block.SZ);
overflow.write(last.buffer, 0, last.count);
}
public void close() throws IOException {
if (overflow != null) {
try {
overflow.close();
} finally {
overflow = null;
}
}
}
/** Clear this buffer so it has no data, and cannot be used again. */
public void destroy() {
blocks = null;
if (overflow != null) {
try {
overflow.close();
} catch (IOException err) {
// We shouldn't encounter an error closing the file.
} finally {
overflow = null;
}
}
}
/**
* A fully buffered output stream using local disk storage for large data.
* <p>
* Initially this output stream buffers to memory and is therefore similar
* to ByteArrayOutputStream, but it shifts to using an on disk temporary
* file if the output gets too large.
* <p>
* The content of this buffered stream may be sent to another OutputStream
* only after this stream has been properly closed by {@link #close()}.
*/
public static class LocalFile extends TemporaryBuffer {
/** Directory to store the temporary file under. */
private final File directory;
/**
* Location of our temporary file if we are on disk; otherwise null.
* <p>
* If we exceeded the {@link #inCoreLimit} we nulled out {@link #blocks}
* and created this file instead. All output goes here through
* {@link #overflow}.
*/
private File onDiskFile;
/**
* Create a new temporary buffer, limiting memory usage.
*
* @param directory
* if the buffer has to spill over into a temporary file, the
* directory where the file should be saved. If null the
* system default temporary directory (for example /tmp) will
* be used instead.
*/
public LocalFile(final File directory) {
this(directory, DEFAULT_IN_CORE_LIMIT);
}
/**
* Create a new temporary buffer, limiting memory usage.
*
* @param directory
* if the buffer has to spill over into a temporary file, the
* directory where the file should be saved. If null the
* system default temporary directory (for example /tmp) will
* be used instead.
* @param inCoreLimit
* maximum number of bytes to store in memory. Storage beyond
* this limit will use the local file.
*/
public LocalFile(final File directory, final int inCoreLimit) {
super(inCoreLimit);
this.directory = directory;
}
protected OutputStream overflow() throws IOException {
onDiskFile = File.createTempFile("jgit_", ".buf", directory); //$NON-NLS-1$ //$NON-NLS-2$
return new BufferedOutputStream(new FileOutputStream(onDiskFile));
}
public long length() {
if (onDiskFile == null) {
return super.length();
}
return onDiskFile.length();
}
public byte[] toByteArray() throws IOException {
if (onDiskFile == null) {
return super.toByteArray();
}
final long len = length();
if (Integer.MAX_VALUE < len)
throw new OutOfMemoryError(JGitText.get().lengthExceedsMaximumArraySize);
final byte[] out = new byte[(int) len];
final FileInputStream in = new FileInputStream(onDiskFile);
try {
IO.readFully(in, out, 0, (int) len);
} finally {
in.close();
}
return out;
}
public void writeTo(final OutputStream os, ProgressMonitor pm)
throws IOException {
if (onDiskFile == null) {
super.writeTo(os, pm);
return;
}
if (pm == null)
pm = NullProgressMonitor.INSTANCE;
final FileInputStream in = new FileInputStream(onDiskFile);
try {
int cnt;
final byte[] buf = new byte[Block.SZ];
while ((cnt = in.read(buf)) >= 0) {
os.write(buf, 0, cnt);
pm.update(cnt / 1024);
}
} finally {
in.close();
}
}
@Override
public InputStream openInputStream() throws IOException {
if (onDiskFile == null)
return super.openInputStream();
return new FileInputStream(onDiskFile);
}
@Override
public void destroy() {
super.destroy();
if (onDiskFile != null) {
try {
if (!onDiskFile.delete())
onDiskFile.deleteOnExit();
} finally {
onDiskFile = null;
}
}
}
}
/**
* A temporary buffer that will never exceed its in-memory limit.
* <p>
* If the in-memory limit is reached an IOException is thrown, rather than
* attempting to spool to local disk.
*/
public static class Heap extends TemporaryBuffer {
/**
* Create a new heap buffer with a maximum storage limit.
*
* @param limit
* maximum number of bytes that can be stored in this buffer;
* also used as the estimated size. Storing beyond this many
* will cause an IOException to be thrown during write.
*/
public Heap(final int limit) {
super(limit);
}
/**
* Create a new heap buffer with a maximum storage limit.
*
* @param estimatedSize
* estimated size of storage used, to size the initial list of
* block pointers.
* @param limit
* maximum number of bytes that can be stored in this buffer.
* Storing beyond this many will cause an IOException to be
* thrown during write.
* @since 4.0
*/
public Heap(final int estimatedSize, final int limit) {
super(estimatedSize, limit);
}
@Override
protected OutputStream overflow() throws IOException {
throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);
}
}
static class Block {
static final int SZ = 8 * 1024;
final byte[] buffer;
int count;
Block() {
buffer = new byte[SZ];
}
Block(int sz) {
buffer = new byte[sz];
}
boolean isFull() {
return count == buffer.length;
}
}
private class BlockInputStream extends InputStream {
private byte[] singleByteBuffer;
private int blockIndex;
private Block block;
private int blockPos;
BlockInputStream() {
block = blocks.get(blockIndex);
}
@Override
public int read() throws IOException {
if (singleByteBuffer == null)
singleByteBuffer = new byte[1];
int n = read(singleByteBuffer);
return n == 1 ? singleByteBuffer[0] & 0xff : -1;
}
@Override
public long skip(long cnt) throws IOException {
long skipped = 0;
while (0 < cnt) {
int n = (int) Math.min(block.count - blockPos, cnt);
if (0 < n) {
blockPos += n;
skipped += n;
cnt -= n;
} else if (nextBlock())
continue;
else
break;
}
return skipped;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (len == 0)
return 0;
int copied = 0;
while (0 < len) {
int c = Math.min(block.count - blockPos, len);
if (0 < c) {
System.arraycopy(block.buffer, blockPos, b, off, c);
blockPos += c;
off += c;
len -= c;
copied += c;
} else if (nextBlock())
continue;
else
break;
}
return 0 < copied ? copied : -1;
}
private boolean nextBlock() {
if (++blockIndex < blocks.size()) {
block = blocks.get(blockIndex);
blockPos = 0;
return true;
}
return false;
}
}
}