/**
* Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.pinot.core.io.writer.impl.v2;
import com.linkedin.pinot.common.segment.ReadMode;
import com.linkedin.pinot.core.io.writer.SingleColumnSingleValueWriter;
import com.linkedin.pinot.core.segment.memory.PinotDataBuffer;
import com.linkedin.pinot.core.util.SizeUtil;
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.util.Arrays;
import me.lemire.integercompression.BitPacking;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Represents a datatable where each col contains values that can be represented
* using a fix set of bits.
*/
public class FixedBitSingleValueWriter implements SingleColumnSingleValueWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(FixedBitSingleValueWriter.class);
private PinotDataBuffer indexDataBuffer;
private int maxValue;
private int minValue;
private int currentRow = -1;
private int numBits;
private int compressedSize;
private int uncompressedSize;
private int[] uncompressedData;
private int[] compressedData;
private int numRows;
private int indexDataBufferPosition = 0;
public FixedBitSingleValueWriter(File file, int rows, int numBits) throws Exception {
init(rows, numBits, false);
createByteBuffer(file);
}
public FixedBitSingleValueWriter(File file, int rows, int numBits,
boolean hasNegativeValues) throws Exception {
init(rows, numBits, hasNegativeValues);
createByteBuffer(file);
}
public FixedBitSingleValueWriter(PinotDataBuffer dataBuffer, int rows, int numBits) throws Exception {
init(rows, numBits, false);
this.indexDataBuffer = dataBuffer;
}
public FixedBitSingleValueWriter(PinotDataBuffer dataBuffer, int rows, int numBits,
boolean hasNegativeValues) throws Exception {
init(rows, numBits, hasNegativeValues);
this.indexDataBuffer = dataBuffer;
}
private void createByteBuffer(File file)
throws IOException {
int bytesRequired = SizeUtil.computeBytesRequired(numRows, numBits, uncompressedSize);
this.indexDataBuffer = PinotDataBuffer.fromFile(file, 0, bytesRequired, ReadMode.mmap, FileChannel.MapMode.READ_WRITE,
file.getAbsolutePath() + this.getClass().getSimpleName());
}
private void init(int rows, int numBits, boolean signed) throws Exception {
this.numRows = rows;
int max = (int) Math.pow(2, numBits);
this.maxValue = max - 1;
// additional bit for sign
if (signed) {
this.minValue = -1 * maxValue;
this.numBits = numBits + 1;
} else {
this.minValue = 0;
this.numBits = numBits;
}
uncompressedSize = SizeUtil.BIT_UNPACK_BATCH_SIZE;
compressedSize = numBits;
uncompressedData = new int[uncompressedSize];
compressedData = new int[compressedSize];
}
public boolean open() {
return true;
}
/**
* @param row
* @param val
*/
public void setInt(int row, int val) {
try {
assert val >= minValue && val <= maxValue && row == currentRow + 1;
int index = row % uncompressedSize;
uncompressedData[index] = val;
if (index == uncompressedSize - 1 || row == numRows - 1) {
BitPacking.fastpack(uncompressedData, 0, compressedData, 0, numBits);
for (int i = 0; i < compressedSize; i++) {
indexDataBuffer.putInt(indexDataBufferPosition, compressedData[i]);
indexDataBufferPosition += 4;
}
int[] out = new int[uncompressedSize];
BitPacking.fastunpack(compressedData, 0, out, 0, numBits);
Arrays.fill(uncompressedData, 0);
}
currentRow = row;
} catch (Exception e) {
LOGGER.error("Failed to set row:{} val:{} ", row, val, e);
throw e;
}
}
@Override
public void close() {
indexDataBuffer.close();
indexDataBuffer = null;
}
@Override
public void setChar(int row, char ch) {
// TODO Auto-generated method stub
}
@Override
public void setShort(int row, short s) {
// TODO Auto-generated method stub
}
@Override
public void setLong(int row, long l) {
// TODO Auto-generated method stub
}
@Override
public void setFloat(int row, float f) {
// TODO Auto-generated method stub
}
@Override
public void setDouble(int row, double d) {
// TODO Auto-generated method stub
}
@Override
public void setString(int row, String string) {
// TODO Auto-generated method stub
}
@Override
public void setBytes(int row, byte[] bytes) {
// TODO Auto-generated method stub
}
}