/**
* Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.pinot.core.segment.index.readers;
import com.linkedin.pinot.common.utils.Pairs.IntPair;
import com.linkedin.pinot.core.segment.memory.PinotDataBuffer;
import java.io.File;
import java.io.IOException;
import java.lang.ref.SoftReference;
import java.nio.ByteBuffer;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BitmapInvertedIndexReader implements InvertedIndexReader {
public static final Logger LOGGER = LoggerFactory.getLogger(BitmapInvertedIndexReader.class);
final private int numberOfBitmaps;
private volatile SoftReference<SoftReference<ImmutableRoaringBitmap>[]> bitmaps = null;
private PinotDataBuffer buffer;
public static final int INT_SIZE_IN_BYTES = Integer.SIZE / Byte.SIZE;
private File file;
/**
* Constructs an inverted index with the specified size.
* @param cardinality the number of bitmaps in the inverted index, which should be the same as the
* number of values in
* the dictionary.
* @throws IOException
*/
public BitmapInvertedIndexReader(PinotDataBuffer indexDataBuffer, int cardinality) throws IOException {
this.file = file;
numberOfBitmaps = cardinality;
load(indexDataBuffer);
}
/**
* {@inheritDoc}
* @see InvertedIndexReader#getImmutable(int)
*/
@Override
public ImmutableRoaringBitmap getImmutable(int idx) {
SoftReference<ImmutableRoaringBitmap>[] bitmapArrayReference = null;
// Return the bitmap if it's still on heap
if (bitmaps != null) {
bitmapArrayReference = bitmaps.get();
if (bitmapArrayReference != null) {
SoftReference<ImmutableRoaringBitmap> bitmapReference = bitmapArrayReference[idx];
if (bitmapReference != null) {
ImmutableRoaringBitmap value = bitmapReference.get();
if (value != null) {
return value;
}
}
} else {
bitmapArrayReference = new SoftReference[numberOfBitmaps];
bitmaps = new SoftReference<SoftReference<ImmutableRoaringBitmap>[]>(bitmapArrayReference);
}
} else {
bitmapArrayReference = new SoftReference[numberOfBitmaps];
bitmaps = new SoftReference<SoftReference<ImmutableRoaringBitmap>[]>(bitmapArrayReference);
}
synchronized (this) {
ImmutableRoaringBitmap value;
if (bitmapArrayReference[idx] == null || bitmapArrayReference[idx].get() == null) {
value = buildRoaringBitmapForIndex(idx);
bitmapArrayReference[idx] = new SoftReference<ImmutableRoaringBitmap>(value);
} else {
value = bitmapArrayReference[idx].get();
}
return value;
}
}
private synchronized ImmutableRoaringBitmap buildRoaringBitmapForIndex(final int index) {
final int currentOffset = getOffset(index);
final int nextOffset = getOffset(index + 1);
final int bufferLength = nextOffset - currentOffset;
// Slice the buffer appropriately for Roaring Bitmap
ByteBuffer bb = buffer.toDirectByteBuffer(currentOffset, bufferLength);
ImmutableRoaringBitmap immutableRoaringBitmap = null;
try {
immutableRoaringBitmap = new ImmutableRoaringBitmap(bb);
} catch (Exception e) {
LOGGER.error(
"Error creating immutableRoaringBitmap for dictionary id:{} currentOffset:{} bufferLength:{} slice position{} limit:{} file:{}",
index, currentOffset, bufferLength, bb.position(), bb.limit(), file.getAbsolutePath());
}
return immutableRoaringBitmap;
}
private int getOffset(final int index) {
return buffer.getInt(index * INT_SIZE_IN_BYTES);
}
private void load(PinotDataBuffer indexDataBuffer) throws IOException {
final int lastOffset = indexDataBuffer.getInt(numberOfBitmaps * INT_SIZE_IN_BYTES);
assert lastOffset == indexDataBuffer.size();
this.buffer = indexDataBuffer;
}
@Override
public void close() throws IOException {
buffer.close();
}
@Override
public IntPair getMinMaxRangeFor(int dicId) {
throw new UnsupportedOperationException("not supported in inverted index type bitmap");
}
}