/**
* Copyright 2011 LiveRamp
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
*
*/
package com.liveramp.hank.storage.cueball;
import com.liveramp.commons.util.BytesUtils;
import com.liveramp.hank.compression.cueball.CueballCompressionCodec;
import com.liveramp.hank.util.IOStreamUtils;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
public final class CueballStreamBuffer {
private final int relativeIndex;
private final int keyHashSize;
private int currentOffset = 0;
private int currentLimit = 0;
private final int fullRecordSize;
private boolean complete;
private final InputStream stream;
private final long[] hashIndex;
private final byte[] uncompressedBuffer;
private final byte[] compressedBuffer;
private int currentHashIndexIdx = -1;
private final CueballCompressionCodec compressionCodec;
private final long dataLength;
public CueballStreamBuffer(String filePath,
int relativeIndex,
int keyHashSize,
int valueSize,
int hashIndexBits,
CueballCompressionCodec compressionCodec)
throws IOException {
this.relativeIndex = relativeIndex;
this.compressionCodec = compressionCodec;
FileInputStream fileInputStream = new FileInputStream(filePath);
this.stream = new BufferedInputStream(fileInputStream, IOStreamUtils.DEFAULT_BUFFER_SIZE);
this.keyHashSize = keyHashSize;
this.fullRecordSize = valueSize + keyHashSize;
Footer footer = new Footer(fileInputStream.getChannel(), hashIndexBits);
dataLength = footer.getDataLength();
hashIndex = footer.getHashIndex();
uncompressedBuffer = new byte[footer.getMaxUncompressedBufferSize()];
compressedBuffer = new byte[footer.getMaxCompressedBufferSize()];
}
public boolean anyRemaining() throws IOException {
if (currentOffset < currentLimit) {
return true;
}
if (complete) {
return false;
}
// refill the buffer
// advance to the next non-empty block
currentHashIndexIdx++;
while (currentHashIndexIdx < hashIndex.length) {
if (hashIndex[currentHashIndexIdx] != -1) {
break;
}
currentHashIndexIdx++;
}
if (currentHashIndexIdx >= hashIndex.length) {
// there are no more blocks. we're all done
complete = true;
return false;
}
// there's at least one block left.
long upperOffset;
if (currentHashIndexIdx == hashIndex.length - 1) {
// actually, there's *exactly* one block left. we need to compare the
// current offset to the data length to determine the compressed block
// size.
upperOffset = dataLength;
} else {
// so this isn't the last block in the index, but it might still be the
// last block in the file.
int nextHashIndexIdx = currentHashIndexIdx + 1;
while (nextHashIndexIdx < hashIndex.length) {
if (hashIndex[nextHashIndexIdx] != -1) {
break;
}
nextHashIndexIdx++;
}
// need to examine the "next" we got back...
if (nextHashIndexIdx == hashIndex.length) {
// turns out that it was the last block in the file, so we should
// compare against the end of the data length.
upperOffset = dataLength;
} else {
// sweet, there's another block there somewhere. use that offset as the
// one to determine our compressed block length.
upperOffset = hashIndex[nextHashIndexIdx];
}
}
final int blockLength = (int) (upperOffset - hashIndex[currentHashIndexIdx]);
// read the compressed block from disk into the compressed buffer
final int compressedBytesRead = stream.read(compressedBuffer, 0, blockLength);
// decompress the compressed block into the uncompressed buffer
final int decompressedSize = compressionCodec.decompress(compressedBuffer, 0, compressedBytesRead, uncompressedBuffer, 0);
// adjust the pointers
currentOffset = 0;
currentLimit = decompressedSize;
return true;
}
/**
* Compare the key hash at the current position of two stream buffers
*
* @param other The stream buffer to compare this to
* @return
*/
public int compareTo(CueballStreamBuffer other) {
return BytesUtils.compareBytesUnsigned(uncompressedBuffer,
currentOffset,
other.uncompressedBuffer,
other.getCurrentOffset(),
keyHashSize);
}
public void consume() {
currentOffset += fullRecordSize;
}
public int getIndex() {
return relativeIndex;
}
public byte[] getBuffer() {
return uncompressedBuffer;
}
public int getCurrentOffset() {
return currentOffset;
}
public void close() throws IOException {
stream.close();
}
}