/**
* Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.pinot.core.io.reader.impl.v1;
import com.linkedin.pinot.core.io.compression.ChunkDecompressor;
import com.linkedin.pinot.core.io.reader.BaseSingleColumnSingleValueReader;
import com.linkedin.pinot.core.io.reader.impl.ChunkReaderContext;
import com.linkedin.pinot.core.segment.memory.PinotDataBuffer;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Abstract class implementation for {@link BaseSingleColumnSingleValueReader}.
* Base class for the fixed and variable byte reader implementations.
*
*/
public abstract class BaseChunkSingleValueReader extends BaseSingleColumnSingleValueReader<ChunkReaderContext> {
private static final Logger LOGGER = LoggerFactory.getLogger(BaseChunkSingleValueReader.class);
protected static final int INT_SIZE = Integer.SIZE / Byte.SIZE;
protected static final int LONG_SIZE = Long.SIZE / Byte.SIZE;
protected static final int FLOAT_SIZE = Float.SIZE / Byte.SIZE;
protected static final int DOUBLE_SIZE = Double.SIZE / Byte.SIZE;
protected final PinotDataBuffer _dataBuffer;
protected final PinotDataBuffer _header;
protected final ChunkDecompressor _chunkDecompressor;
protected final int _chunkSize;
protected final int _numDocsPerChunk;
protected final int _numChunks;
protected final int _lengthOfLongestEntry;
/**
* Constructor for the class.
*
* @param pinotDataBuffer Data buffer
* @param decompressor Data decompressor
*/
public BaseChunkSingleValueReader(PinotDataBuffer pinotDataBuffer, ChunkDecompressor decompressor) {
_chunkDecompressor = decompressor;
_dataBuffer = pinotDataBuffer;
int headerOffset = INT_SIZE; // First entry is the version, which is unused currently.
_numChunks = _dataBuffer.getInt(headerOffset);
headerOffset += INT_SIZE;
_numDocsPerChunk = _dataBuffer.getInt(headerOffset);
headerOffset += INT_SIZE;
_lengthOfLongestEntry = _dataBuffer.getInt(headerOffset);
headerOffset += INT_SIZE;
_chunkSize = (_lengthOfLongestEntry * _numDocsPerChunk);
// Slice out the header from the data buffer.
int headerLength = _numChunks * INT_SIZE;
_header = _dataBuffer.view(headerOffset, headerOffset + headerLength);
}
@Override
public void close() {
// Nothing to close here.
}
/**
* Helper method to get the chunk for a given row.
* <ul>
* <li> If the chunk already exists in the reader context, returns the same. </li>
* <li> Otherwise, loads the chunk for the row, and sets it in the reader context. </li>
* </ul>
* @param row Row for which to get the chunk
* @param context Reader context
* @return Chunk for the row
*/
protected ByteBuffer getChunkForRow(int row, ChunkReaderContext context) {
int chunkId = row / _numDocsPerChunk;
if (context.getChunkId() == chunkId) {
return context.getChunkBuffer();
}
int chunkSize;
int chunkPosition = getChunkPosition(chunkId);
// Size of chunk can be determined using next chunks offset, or end of data buffer for last chunk.
if (chunkId == (_numChunks - 1)) { // Last chunk.
chunkSize = (int) (_dataBuffer.size() - chunkPosition);
} else {
int nextChunkOffset = getChunkPosition(chunkId + 1);
chunkSize = nextChunkOffset - chunkPosition;
}
ByteBuffer uncompressedBuffer = context.getChunkBuffer();
uncompressedBuffer.clear();
try {
_chunkDecompressor.decompress(_dataBuffer.toDirectByteBuffer(chunkPosition, chunkSize), uncompressedBuffer);
} catch (IOException e) {
LOGGER.error("Exception caught while decompressing data chunk", e);
throw new RuntimeException(e);
}
context.setChunkId(chunkId);
return uncompressedBuffer;
}
/**
* Helper method to get the offset of the chunk in the data.
*
* @param chunkId Id of the chunk for which to return the position.
* @return Position (offset) of the chunk in the data.
*/
protected int getChunkPosition(int chunkId) {
return _header.getInt(chunkId * INT_SIZE);
}
}