/*
* Copyright 2015 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.io.orc;
import java.io.File;
import java.io.IOException;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileSplit;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
public class TestOrcFileStripeMergeRecordReader {
private final int DEFAULT_STRIPE_SIZE = 5000;
private OrcFileKeyWrapper key;
private OrcFileValueWrapper value;
private Path tmpPath;
private Configuration conf;
private FileSystem fs;
@Rule
public TestName testCaseName = new TestName();
@Before
public void setup() throws IOException {
conf = new Configuration();
fs = FileSystem.getLocal(conf);
key = new OrcFileKeyWrapper();
value = new OrcFileValueWrapper();
tmpPath = prepareTmpPath();
}
@Test
public void testSplitStartsWithOffset() throws IOException {
createOrcFile(DEFAULT_STRIPE_SIZE, DEFAULT_STRIPE_SIZE + 1);
FileStatus fileStatus = fs.getFileStatus(tmpPath);
long length = fileStatus.getLen();
long offset = length / 2;
FileSplit split = new FileSplit(tmpPath, offset, length, (String[])null);
OrcFileStripeMergeRecordReader reader = new OrcFileStripeMergeRecordReader(conf, split);
reader.next(key, value);
Assert.assertEquals("InputPath", tmpPath, key.getInputPath());
Assert.assertEquals("NumberOfValues", 1L, value.getStripeStatistics().getColStats(0).getNumberOfValues());
reader.close();
}
private void createOrcFile(int stripSize, int numberOfRows) throws IOException {
ObjectInspector inspector;
synchronized (TestOrcFileStripeMergeRecordReader.class) {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(StringIntIntIntRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
Writer writer = OrcFile.createWriter(tmpPath,
OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(stripSize)
.compress(CompressionKind.ZLIB)
.bufferSize(5000)
.rowIndexStride(1000));
Random rand = new Random(157);
for (int i = 0; i < numberOfRows; i++) {
writer.addRow(new StringIntIntIntRow(
Integer.toBinaryString(i),
rand.nextInt(),
rand.nextInt(),
rand.nextInt()
));
}
writer.close();
}
private Path prepareTmpPath() throws IOException {
Path path = new Path(System.getProperty("test.tmp.dir",
"target" + File.separator + "test" + File.separator + "tmp")
+ File.separator + "TestOrcFile." + testCaseName.getMethodName() + ".orc");
fs.delete(path, false);
return path;
}
public static class StringIntIntIntRow {
Text string1 = new Text();
int int1;
int int2;
int int3;
StringIntIntIntRow(String string1, int int1, int int2, int int3) {
this.string1.set(string1);
this.int1 = int1;
this.int2 = int2;
this.int3 = int3;
}
}
}