/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.io;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.CharBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.text.NumberFormat;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.IOUtils;
import org.archive.util.DevUtils;
import com.google.common.base.Charsets;
import com.google.common.primitives.Ints;
/**
* (Replay)CharSequence view on recorded streams.
*
* For small streams, use {@link InMemoryReplayCharSequence}.
*
* <p>Call {@link close()} on this class when done to clean up resources.
*
* @contributor stack
* @contributor nlevitt
* @version $Revision$, $Date$
*/
public class GenericReplayCharSequence implements ReplayCharSequence {
protected static Logger logger = Logger
.getLogger(GenericReplayCharSequence.class.getName());
/**
* Name of the encoding we use writing out concatenated decoded prefix
* buffer and decoded backing file.
*
* <p>This define is also used as suffix for the file that holds the
* decodings. The name of the file that holds the decoding is the name
* of the backing file w/ this encoding for a suffix.
*
* <p>See <a ref="http://java.sun.com/j2se/1.4.2/docs/guide/intl/encoding.doc.html">Encoding</a>.
*/
public static final Charset WRITE_ENCODING = Charsets.UTF_16BE;
private static final long MAP_MAX_BYTES = 64 * 1024 * 1024; // 64M
/**
* When the memory map moves away from the beginning of the file
* (to the "right") in order to reach a certain index, it will
* map up to this many bytes preceding (to the left of) the target character.
* Consequently it will map up to
* <code>MAP_MAX_BYTES - MAP_TARGET_LEFT_PADDING</code>
* bytes to the right of the target.
*/
private static final long MAP_TARGET_LEFT_PADDING_BYTES = (long) (MAP_MAX_BYTES * 0.01);
/**
* Total length of character stream to replay minus the HTTP headers
* if present.
*
* If the backing file is larger than <code>Integer.MAX_VALUE</code> (i.e. 2gb),
* only the first <code>Integer.MAX_VALUE</code> characters are available through this API.
* We're overriding <code>java.lang.CharSequence</code> so that we can use
* <code>java.util.regex</code> directly on the data, and the <code>CharSequence</code>
* API uses <code>int</code> for the length and index.
*/
protected int length;
/** counter of decoding exceptions for report at end */
protected long decodingExceptions = 0;
protected CharacterCodingException codingException = null;
/**
* Byte offset into the file where the memory mapped portion begins.
*/
private long mapByteOffset;
// XXX do we need to keep the input stream around?
private FileInputStream backingFileIn = null;
private FileChannel backingFileChannel = null;
private long bytesPerChar;
private CharBuffer mappedBuffer = null;
/**
* File that has decoded content.
*
* Keep it around so we can remove on close.
*/
private File decodedFile = null;
/*
* This portion of the CharSequence precedes what's in the backing file. In
* cases where we decodeToFile(), this is always empty, because we decode
* the entire input stream.
*/
private CharBuffer prefixBuffer = null;
private boolean isOpen = true;
protected Charset charset = null;
/**
* Constructor.
*
* @param contentReplayInputStream inputStream of content
* @param charset Encoding to use reading the passed prefix
* buffer and backing file. Must not be null.
* @param backingFilename Path to backing file with content in excess of
* whats in <code>buffer</code>.
*
* @throws IOException
*/
public GenericReplayCharSequence(InputStream contentReplayInputStream,
int prefixMax,
String backingFilename,
Charset charset) throws IOException {
super();
logger.fine("characterEncoding=" + charset + " backingFilename="
+ backingFilename);
if(charset==null) {
charset = ReplayCharSequence.FALLBACK_CHARSET;
}
// decodes only up to Integer.MAX_VALUE characters
decode(contentReplayInputStream, prefixMax, backingFilename, charset);
this.bytesPerChar = 2;
if(length>prefixBuffer.position()) {
this.backingFileIn = new FileInputStream(decodedFile);
this.backingFileChannel = backingFileIn.getChannel();
this.mapByteOffset = 0;
updateMemoryMappedBuffer();
}
}
private void updateMemoryMappedBuffer() {
long charLength = (long) this.length() - (long) prefixBuffer.limit(); // in characters
long mapSize = Math.min((charLength * bytesPerChar) - mapByteOffset, MAP_MAX_BYTES);
logger.fine("updateMemoryMappedBuffer: mapOffset="
+ NumberFormat.getInstance().format(mapByteOffset)
+ " mapSize=" + NumberFormat.getInstance().format(mapSize));
try {
// TODO: stress-test without these possibly-costly requests!
// System.gc();
// System.runFinalization();
// TODO: Confirm the READ_ONLY works. I recall it not working.
// The buffers seem to always say that the buffer is writable.
mappedBuffer = backingFileChannel.map(
FileChannel.MapMode.READ_ONLY, mapByteOffset, mapSize)
.asReadOnlyBuffer().asCharBuffer();
} catch (IOException e) {
// TODO convert this to a runtime error?
DevUtils.logger.log(Level.SEVERE,
" backingFileChannel.map() mapByteOffset=" + mapByteOffset
+ " mapSize=" + mapSize + "\n" + "decodedFile="
+ decodedFile + " length=" + length + "\n"
+ DevUtils.extraInfo(), e);
throw new RuntimeException(e);
}
}
/**
* Converts the first <code>Integer.MAX_VALUE</code> characters from the
* file <code>backingFilename</code> from encoding <code>encoding</code> to
* encoding <code>WRITE_ENCODING</code> and saves as
* <code>this.decodedFile</code>, which is named <code>backingFilename
* + "." + WRITE_ENCODING</code>.
*
* @throws IOException
*/
protected void decode(InputStream inStream, int prefixMax,
String backingFilename, Charset charset) throws IOException {
this.charset = charset;
// TODO: consider if BufferedReader is helping any
// TODO: consider adding TBW 'LimitReader' to stop reading at
// Integer.MAX_VALUE characters because of charAt(int) limit
BufferedReader reader = new BufferedReader(new InputStreamReader(
inStream, charset));
logger.fine("backingFilename=" + backingFilename + " encoding="
+ charset + " decodedFile=" + decodedFile);
this.prefixBuffer = CharBuffer.allocate(prefixMax);
long count = 0;
while(count < prefixMax) {
int read = reader.read(prefixBuffer);
if(read<0) {
break;
}
count += read;
}
int ch = reader.read();
if(ch >= 0) {
count++;
// more to decode to file overflow
this.decodedFile = new File(backingFilename + "." + WRITE_ENCODING);
FileOutputStream fos;
try {
fos = new FileOutputStream(this.decodedFile);
} catch (FileNotFoundException e) {
// Windows workaround attempt
System.gc();
System.runFinalization();
this.decodedFile = new File(decodedFile.getAbsolutePath()+".win");
logger.info("Windows 'file with a user-mapped section open' "
+ "workaround gc/finalization/name-extension performed.");
// try again
fos = new FileOutputStream(this.decodedFile);
}
Writer writer = new OutputStreamWriter(fos,WRITE_ENCODING);
writer.write(ch);
count += IOUtils.copyLarge(reader, writer);
writer.close();
reader.close();
}
this.length = Ints.saturatedCast(count);
if(count>Integer.MAX_VALUE) {
logger.warning("input stream is longer than Integer.MAX_VALUE="
+ NumberFormat.getInstance().format(Integer.MAX_VALUE)
+ " characters -- only first "
+ NumberFormat.getInstance().format(Integer.MAX_VALUE)
+ " are accessible through this GenericReplayCharSequence");
}
logger.fine("decode: decoded " + count + " characters" +
((decodedFile==null) ? ""
: " ("+(count-prefixBuffer.length())+" to "+decodedFile+")"));
}
/**
* Get character at passed absolute position.
* @param index Index into content
* @return Character at offset <code>index</code>.
*/
public char charAt(int index) {
if (index < 0 || index >= this.length()) {
throw new IndexOutOfBoundsException("index=" + index
+ " - should be between 0 and length()=" + this.length());
}
// is it in the buffer
if (index < prefixBuffer.limit()) {
return prefixBuffer.get(index);
}
// otherwise we gotta get it from disk via memory map
long charFileIndex = (long) index - (long) prefixBuffer.limit();
long charFileLength = (long) this.length() - (long) prefixBuffer.limit(); // in characters
if (charFileIndex * bytesPerChar < mapByteOffset) {
logger.log(Level.WARNING,"left-fault; probably don't want to use CharSequence that far backward");
}
if (charFileIndex * bytesPerChar < mapByteOffset
|| charFileIndex - (mapByteOffset / bytesPerChar) >= mappedBuffer.limit()) {
// fault
/*
* mapByteOffset is bounded by 0 and file size +/- size of the map,
* and starts as close to <code>fileIndex -
* MAP_TARGET_LEFT_PADDING_BYTES</code> as it can while also not
* being smaller than it needs to be.
*/
mapByteOffset = Math.min(charFileIndex * bytesPerChar - MAP_TARGET_LEFT_PADDING_BYTES,
charFileLength * bytesPerChar - MAP_MAX_BYTES);
mapByteOffset = Math.max(0, mapByteOffset);
updateMemoryMappedBuffer();
}
return mappedBuffer.get((int)(charFileIndex-(mapByteOffset/bytesPerChar)));
}
public CharSequence subSequence(int start, int end) {
return new CharSubSequence(this, start, end);
}
private void deleteFile(File fileToDelete) {
deleteFile(fileToDelete, null);
}
private void deleteFile(File fileToDelete, final Exception e) {
if (e != null) {
// Log why the delete to help with debug of
// java.io.FileNotFoundException:
// ....tt53http.ris.UTF-16BE.
logger.severe("Deleting " + fileToDelete + " because of "
+ e.toString());
}
if (fileToDelete != null && fileToDelete.exists()) {
logger.fine("deleting file: " + fileToDelete);
fileToDelete.delete();
}
}
@Override
public boolean isOpen() {
return this.isOpen;
}
public void close() throws IOException {
this.isOpen = false;
logger.fine("closing");
if (this.backingFileChannel != null && this.backingFileChannel.isOpen()) {
this.backingFileChannel.close();
}
if (backingFileIn != null) {
backingFileIn.close();
}
deleteFile(this.decodedFile);
// clear decodedFile -- so that double-close (as in finalize()) won't
// delete a later instance with same name see bug [ 1218961 ]
// "failed get of replay" in ExtractorHTML... usu: UTF-16BE
this.decodedFile = null;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#finalize()
*/
protected void finalize() throws Throwable {
super.finalize();
logger.fine("finalizing");
close();
}
/**
* Convenience method for getting a substring.
*
* @deprecated please use subSequence() and then toString() directly
*/
public String substring(int offset, int len) {
return subSequence(offset, offset + len).toString();
}
public String toString() {
StringBuilder sb = new StringBuilder(this.length());
sb.append(this);
return sb.toString();
}
public int length() {
return length;
}
/* (non-Javadoc)
* @see org.archive.io.ReplayCharSequence#getDecodeExceptionCount()
*/
@Override
public long getDecodeExceptionCount() {
return decodingExceptions;
}
/* (non-Javadoc)
* @see org.archive.io.ReplayCharSequence#getCodingException()
*/
@Override
public CharacterCodingException getCodingException() {
return codingException;
}
/* (non-Javadoc)
* @see org.archive.io.ReplayCharSequence#getCharset()
*/
public Charset getCharset() {
return charset;
}
}