/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.io;
import java.io.IOException;
import java.io.InputStream;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.security.MessageDigest;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.IOUtils;
/**
* Stream which records all data read from it, which it acquires from a wrapped
* input stream.
*
* Makes use of a RecordingOutputStream for recording because of its being
* file backed so we can write massive amounts of data w/o worrying about
* overflowing memory.
*
* @author gojomo
*
*/
public class RecordingInputStream
extends InputStream {
protected static Logger logger =
Logger.getLogger("org.archive.io.RecordingInputStream");
/**
* Where we are recording to.
*/
private RecordingOutputStream recordingOutputStream;
/**
* Stream to record.
*/
private InputStream in = null;
/**
* Reusable buffer to avoid reallocation on each readFullyUntil
*/
protected byte[] drainBuffer = new byte[16*1024];
/**
* Create a new RecordingInputStream.
*
* @param bufferSize Size of buffer to use.
* @param backingFilename Name of backing file.
*/
public RecordingInputStream(int bufferSize, String backingFilename)
{
this.recordingOutputStream = new RecordingOutputStream(bufferSize,
backingFilename);
}
public void open(InputStream wrappedStream) throws IOException {
if (logger.isLoggable(Level.FINE)) {
logger.fine("wrapping " + wrappedStream + " in thread "
+ Thread.currentThread().getName());
}
if(isOpen()) {
// error; should not be opening/wrapping in an unclosed
// stream remains open
throw new IOException("RIS already open for "
+Thread.currentThread().getName());
}
try {
this.in = wrappedStream;
this.recordingOutputStream.open();
} catch (IOException ioe) {
close(); // ...and rethrow...
throw ioe;
}
}
public int read() throws IOException {
if (!isOpen()) {
throw new IOException("Stream closed " +
Thread.currentThread().getName());
}
int b = this.in.read();
if (b != -1) {
assert this.recordingOutputStream != null: "ROS is null " +
Thread.currentThread().getName();
this.recordingOutputStream.write(b);
}
return b;
}
public int read(byte[] b, int off, int len) throws IOException {
if (!isOpen()) {
throw new IOException("Stream closed " +
Thread.currentThread().getName());
}
int count = this.in.read(b,off,len);
if (count > 0) {
assert this.recordingOutputStream != null: "ROS is null " +
Thread.currentThread().getName();
this.recordingOutputStream.write(b,off,count);
}
return count;
}
public int read(byte[] b) throws IOException {
if (!isOpen()) {
throw new IOException("Stream closed " +
Thread.currentThread().getName());
}
int count = this.in.read(b);
if (count > 0) {
assert this.recordingOutputStream != null: "ROS is null " +
Thread.currentThread().getName();
this.recordingOutputStream.write(b,0,count);
}
return count;
}
public void close() throws IOException {
if (logger.isLoggable(Level.FINE)) {
logger.fine("closing " + this.in + " in thread "
+ Thread.currentThread().getName());
}
IOUtils.closeQuietly(this.in);
this.in = null;
IOUtils.closeQuietly(this.recordingOutputStream);
}
public ReplayInputStream getReplayInputStream() throws IOException {
return this.recordingOutputStream.getReplayInputStream();
}
public ReplayInputStream getMessageBodyReplayInputStream() throws IOException {
return this.recordingOutputStream.getMessageBodyReplayInputStream();
}
public long readFully() throws IOException {
while(read(drainBuffer) != -1) {
// Empty out stream.
continue;
}
return this.recordingOutputStream.getSize();
}
public void readToEndOfContent(long contentLength)
throws IOException, InterruptedException {
// Check we're open before proceeding.
if (!isOpen()) {
// TODO: should this be a noisier exception-raising error?
return;
}
long totalBytes = recordingOutputStream.position - recordingOutputStream.getMessageBodyBegin();
long bytesRead = -1L;
long maxToRead = -1;
while (contentLength <= 0 || totalBytes < contentLength) {
try {
// read no more than soft max
maxToRead = (contentLength <= 0)
? drainBuffer.length
: Math.min(drainBuffer.length, contentLength - totalBytes);
// nor more than hard max
maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength());
// but always at least 1 (to trigger hard max exception) XXX wtf is this?
maxToRead = Math.max(maxToRead, 1);
bytesRead = read(drainBuffer,0,(int)maxToRead);
if (bytesRead == -1) {
break;
}
totalBytes += bytesRead;
if (Thread.interrupted()) {
throw new InterruptedException("Interrupted during IO");
}
} catch (SocketTimeoutException e) {
// A socket timeout is just a transient problem, meaning
// nothing was available in the configured timeout period,
// but something else might become available later.
// Take this opportunity to check the overall
// timeout (below). One reason for this timeout is
// servers that keep up the connection, 'keep-alive', even
// though we asked them to not keep the connection open.
if (logger.isLoggable(Level.FINE)) {
logger.log(Level.FINE, "socket timeout", e);
}
// check for interrupt
if (Thread.interrupted()) {
throw new InterruptedException("Interrupted during IO");
}
// check for overall timeout
recordingOutputStream.checkLimits();
} catch (SocketException se) {
throw se;
} catch (NullPointerException e) {
// [ 896757 ] NPEs in Andy's Th-Fri Crawl.
// A crawl was showing NPE's in this part of the code but can
// not reproduce. Adding this rethrowing catch block w/
// diagnostics to help should we come across the problem in the
// future.
throw new NullPointerException("Stream " + this.in + ", " +
e.getMessage() + " " + Thread.currentThread().getName());
}
}
}
/**
* Read all of a stream (Or read until we timeout or have read to the max).
* @param softMaxLength Maximum length to read; if zero or < 0, then no
* limit. If met, return normally.
* @throws IOException failed read.
* @throws RecorderLengthExceededException
* @throws RecorderTimeoutException
* @throws InterruptedException
* @deprecated
*/
public void readFullyOrUntil(long softMaxLength)
throws IOException, RecorderLengthExceededException,
RecorderTimeoutException, InterruptedException {
// Check we're open before proceeding.
if (!isOpen()) {
// TODO: should this be a noisier exception-raising error?
return;
}
long totalBytes = 0L;
long bytesRead = -1L;
long maxToRead = -1;
while (true) {
try {
// read no more than soft max
maxToRead = (softMaxLength <= 0)
? drainBuffer.length
: Math.min(drainBuffer.length, softMaxLength - totalBytes);
// nor more than hard max
maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength());
// but always at least 1 (to trigger hard max exception
maxToRead = Math.max(maxToRead, 1);
bytesRead = read(drainBuffer,0,(int)maxToRead);
if (bytesRead == -1) {
break;
}
totalBytes += bytesRead;
if (Thread.interrupted()) {
throw new InterruptedException("Interrupted during IO");
}
} catch (SocketTimeoutException e) {
// A socket timeout is just a transient problem, meaning
// nothing was available in the configured timeout period,
// but something else might become available later.
// Take this opportunity to check the overall
// timeout (below). One reason for this timeout is
// servers that keep up the connection, 'keep-alive', even
// though we asked them to not keep the connection open.
if (logger.isLoggable(Level.FINE)) {
logger.log(Level.FINE, "socket timeout", e);
}
// check for interrupt
if (Thread.interrupted()) {
throw new InterruptedException("Interrupted during IO");
}
// check for overall timeout
recordingOutputStream.checkLimits();
} catch (SocketException se) {
throw se;
} catch (NullPointerException e) {
// [ 896757 ] NPEs in Andy's Th-Fri Crawl.
// A crawl was showing NPE's in this part of the code but can
// not reproduce. Adding this rethrowing catch block w/
// diagnostics to help should we come across the problem in the
// future.
throw new NullPointerException("Stream " + this.in + ", " +
e.getMessage() + " " + Thread.currentThread().getName());
}
// if have read 'enough', just finish
if (softMaxLength > 0 && totalBytes >= softMaxLength) {
break; // return
}
}
}
public long getSize() {
return this.recordingOutputStream.getSize();
}
public void markContentBegin() {
this.recordingOutputStream.markMessageBodyBegin();
}
public long getContentBegin() {
return this.recordingOutputStream.getMessageBodyBegin();
}
public void startDigest() {
this.recordingOutputStream.startDigest();
}
/**
* Convenience method for setting SHA1 digest.
*/
public void setSha1Digest() {
this.recordingOutputStream.setSha1Digest();
}
/**
* Sets a digest algorithm which may be applied to recorded data.
* As usually only a subset of the recorded data should
* be fed to the digest, you must also call startDigest()
* to begin digesting.
*
* @param algorithm
*/
public void setDigest(String algorithm) {
this.recordingOutputStream.setDigest(algorithm);
}
/**
* Sets a digest function which may be applied to recorded data.
* As usually only a subset of the recorded data should
* be fed to the digest, you must also call startDigest()
* to begin digesting.
*
* @param md
*/
public void setDigest(MessageDigest md) {
this.recordingOutputStream.setDigest(md);
}
/**
* Return the digest value for any recorded, digested data. Call
* only after all data has been recorded; otherwise, the running
* digest state is ruined.
*
* @return the digest final value
*/
public byte[] getDigestValue() {
return this.recordingOutputStream.getDigestValue();
}
public long getResponseContentLength() {
return this.recordingOutputStream.getResponseContentLength();
}
public void closeRecorder() throws IOException {
this.recordingOutputStream.closeRecorder();
}
/**
* @return True if we've been opened.
*/
public boolean isOpen()
{
return this.in != null;
}
@Override
public synchronized void mark(int readlimit) {
this.in.mark(readlimit);
this.recordingOutputStream.mark();
}
@Override
public boolean markSupported() {
return this.in.markSupported();
}
@Override
public synchronized void reset() throws IOException {
this.in.reset();
this.recordingOutputStream.reset();
}
/**
* Set limits to be enforced by internal recording-out
*/
public void setLimits(long hardMax, long timeoutMs, long maxRateKBps) {
recordingOutputStream.setLimits(hardMax, timeoutMs, maxRateKBps);
}
/**
* Expose the amount of in-memory buffering used by the internal
* recording stream.
* @return int buffer size
*/
public int getRecordedBufferLength() {
return recordingOutputStream.getBufferLength();
}
/**
* See doc on {@link RecordingOutputStream#chopAtMessageBodyBegin()}
*/
public void chopAtMessageBodyBegin() {
recordingOutputStream.chopAtMessageBodyBegin();
}
public void clearForReuse() throws IOException {
recordingOutputStream.clearForReuse();
}
}