/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.core.io.writer.impl; import com.linkedin.pinot.common.segment.ReadMode; import com.linkedin.pinot.core.segment.memory.PinotDataBuffer; import com.linkedin.pinot.core.util.PinotDataCustomBitSet; import java.io.Closeable; import java.io.File; import java.nio.channels.FileChannel; import java.util.Arrays; /** * Represents a datatable where each col contains values that can be represented * using a fix set of bits. * * */ public class FixedBitSingleValueMultiColWriter implements Closeable { private int[] columnOffsetsInBits; private int[] offsets; private PinotDataBuffer indexDataBuffer; private int rowSizeInBits; private int[] colSizesInBits; private int[] maxValues; private int[] minValues; private PinotDataCustomBitSet bitSet; private int bytesRequired; public FixedBitSingleValueMultiColWriter(File file, int rows, int cols, int[] columnSizesInBits) throws Exception { init(rows, cols, columnSizesInBits); this.indexDataBuffer = PinotDataBuffer.fromFile(file, 0, bytesRequired, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, file.getAbsolutePath() + this.getClass().getCanonicalName()); bitSet = PinotDataCustomBitSet.withDataBuffer(bytesRequired, indexDataBuffer); } public FixedBitSingleValueMultiColWriter(File file, int rows, int cols, int[] columnSizesInBits, boolean[] hasNegativeValues) throws Exception { init(rows, cols, columnSizesInBits, hasNegativeValues); this.indexDataBuffer = PinotDataBuffer.fromFile(file, 0, bytesRequired, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, file.getAbsolutePath() + this.getClass().getCanonicalName()); bitSet = PinotDataCustomBitSet.withDataBuffer(bytesRequired, indexDataBuffer); } public FixedBitSingleValueMultiColWriter(PinotDataBuffer dataBuffer, int rows, int cols, int[] columnSizesInBits) throws Exception { init(rows, cols, columnSizesInBits); this.indexDataBuffer = dataBuffer; bitSet = PinotDataCustomBitSet.withDataBuffer(bytesRequired, indexDataBuffer); } public FixedBitSingleValueMultiColWriter(PinotDataBuffer dataBuffer, int rows, int cols, int[] columnSizesInBits, boolean[] hasNegativeValues) throws Exception { init(rows, cols, columnSizesInBits, hasNegativeValues); this.indexDataBuffer = dataBuffer; bitSet = PinotDataCustomBitSet.withDataBuffer(bytesRequired, indexDataBuffer); } private void init(int rows, int cols, int[] columnSizesInBits) { boolean[] hasNegativeValues = new boolean[cols]; Arrays.fill(hasNegativeValues, false); init(rows, cols, columnSizesInBits, hasNegativeValues); } private void init(int rows, int cols, int[] columnSizesInBits, boolean[] signed) { this.colSizesInBits = new int[cols]; this.columnOffsetsInBits = new int[cols]; this.offsets = new int[cols]; this.maxValues = new int[cols]; this.minValues = new int[cols]; this.rowSizeInBits = 0; for (int i = 0; i < cols; i++) { this.columnOffsetsInBits[i] = rowSizeInBits; int colSize = columnSizesInBits[i]; int max = (int) Math.pow(2, colSize); this.maxValues[i] = max - 1; this.minValues[i] = 0; this.offsets[i] = 0; // additional bit for sign if (signed[i]) { this.offsets[i] = this.maxValues[i]; this.minValues[i] = -this.maxValues[i]; colSize += 1; } this.rowSizeInBits += colSize; this.colSizesInBits[i] = colSize; } long totalSizeInBits = ((long) rowSizeInBits) * rows; // jfim: We keep the number of bytes required as an int, as Java Buffers cannot be larger than 2GB this.bytesRequired = (int)((totalSizeInBits + 7) / 8); } public boolean open() { return true; } /** * * @param row * @param col * @param val */ public void setInt(int row, int col, int val) { assert val >= minValues[col] && val <= maxValues[col]; long bitOffset = ((long) rowSizeInBits) * row + columnOffsetsInBits[col]; val = val + offsets[col]; bitSet.writeInt(bitOffset, colSizesInBits[col], val); } @Override public void close() { bitSet.close(); indexDataBuffer.close(); bitSet = null; indexDataBuffer = null; } }