/**
* Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.pinot.index.reader;
import com.linkedin.pinot.common.segment.ReadMode;
import com.linkedin.pinot.core.segment.memory.PinotDataBuffer;
import java.io.File;
import java.nio.channels.FileChannel;
import java.util.Random;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
import com.linkedin.pinot.core.io.reader.impl.FixedByteSingleValueMultiColReader;
import com.linkedin.pinot.core.io.reader.impl.SortedForwardIndexReader;
import com.linkedin.pinot.core.io.reader.impl.SortedValueReaderContext;
import com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter;
@Test
public class SortedForwardIndexReaderTest {
private static Logger LOGGER = LoggerFactory.getLogger(SortedForwardIndexReaderTest.class);
public void testSimple() throws Exception {
int maxLength = 1000;
int cardinality = 100000;
File file = new File("test_sortef_fwd_index.dat");
file.delete();
int[] columnSizes = new int[] {
4, 4
};
FixedByteSingleValueMultiColWriter writer =
new FixedByteSingleValueMultiColWriter(file, cardinality, columnSizes.length, columnSizes);
Random random = new Random();
int[] startDocIdArray = new int[cardinality];
int[] endDocIdArray = new int[cardinality];
int prevEnd = -1;
int totalDocs = 0;
for (int i = 0; i < cardinality; i++) {
int length = random.nextInt(maxLength);
int start = prevEnd + 1;
int end = start + length;
startDocIdArray[i] = start;
endDocIdArray[i] = end;
writer.setInt(i, 0, start);
writer.setInt(i, 1, end);
prevEnd = end;
totalDocs += length;
}
writer.close();
PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
FixedByteSingleValueMultiColReader rawFileReader = new FixedByteSingleValueMultiColReader(heapBuffer,
cardinality, columnSizes);
SortedForwardIndexReader reader = new SortedForwardIndexReader(rawFileReader, totalDocs);
// without using context
long start, end;
start = System.currentTimeMillis();
for (int i = 0; i < cardinality; i++) {
for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
Assert.assertEquals(reader.getInt(docId), i);
}
}
end = System.currentTimeMillis();
System.out
.println("Took " + (end - start) + " to scan " + totalDocs + " docs without using context");
// with context
SortedValueReaderContext context = reader.createContext();
start = System.currentTimeMillis();
for (int i = 0; i < cardinality; i++) {
for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
Assert.assertEquals(reader.getInt(docId, context), i);
}
}
end = System.currentTimeMillis();
LOGGER.debug("Took " + (end - start) + " to scan " + totalDocs + " with context");
reader.close();
file.delete();
heapBuffer.close();
}
}