package org.apache.blur.mapreduce.lib;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.TreeSet;
import org.apache.blur.MiniCluster;
import org.apache.blur.server.TableContext;
import org.apache.blur.store.buffer.BufferStore;
import org.apache.blur.store.hdfs.HdfsDirectory;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.blur.utils.JavaHome;
import org.apache.blur.utils.ShardUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.lucene.index.DirectoryReader;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
public class BlurOutputFormatTest {
private static Configuration _conf = new Configuration();
private static FileSystem _fileSystem;
private static MiniCluster _miniCluster;
private static Path _root;
@BeforeClass
public static void setupTest() throws Exception {
JavaHome.checkJavaHome();
File file = new File("./target/tmp/BlurOutputFormatTest_tmp");
String pathStr = file.getAbsoluteFile().toURI().toString();
String hdfsPath = pathStr + "/hdfs";
System.setProperty("test.build.data", hdfsPath);
System.setProperty("hadoop.log.dir", pathStr + "/hadoop_log");
_miniCluster = new MiniCluster();
_miniCluster.startDfs(hdfsPath);
_fileSystem = _miniCluster.getFileSystem();
_root = new Path(_fileSystem.getUri() + "/testroot");
_miniCluster.startMrMiniCluster();
_conf = _miniCluster.getMRConfiguration();
BufferStore.initNewBuffer(128, 128 * 128);
}
@AfterClass
public static void teardown() throws IOException {
if (_miniCluster != null) {
_miniCluster.stopMrMiniCluster();
_miniCluster.shutdownDfs();
}
rm(new File("build"));
}
private static void rm(File file) {
if (!file.exists()) {
return;
}
if (file.isDirectory()) {
for (File f : file.listFiles()) {
rm(f);
}
}
file.delete();
}
@Before
public void setup() {
TableContext.clear();
}
@Test
public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException {
Path input = getInDir();
Path output = getOutDir();
_fileSystem.delete(input, true);
_fileSystem.delete(output, true);
writeRecordsFile(new Path(input, "part1"), 1, 1, 1, 1, "cf1");
writeRecordsFile(new Path(input, "part2"), 1, 1, 2, 1, "cf1");
Job job = Job.getInstance(_conf, "blur index");
job.setJarByClass(BlurOutputFormatTest.class);
job.setMapperClass(CsvBlurMapper.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, input);
CsvBlurMapper.addColumns(job, "cf1", "col");
Path tablePath = new Path(new Path(_root, "table"), "test");
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setShardCount(1);
tableDescriptor.setTableUri(tablePath.toString());
tableDescriptor.setName("test");
createShardDirectories(tablePath, 1);
BlurOutputFormat.setupJob(job, tableDescriptor);
BlurOutputFormat.setOutputPath(job, output);
assertTrue(job.waitForCompletion(true));
Counters ctrs = job.getCounters();
System.out.println("Counters: " + ctrs);
Path path = new Path(output, ShardUtil.getShardName(0));
dump(path, _conf);
Collection<Path> commitedTasks = getCommitedTasks(path);
assertEquals(1, commitedTasks.size());
DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(_conf, commitedTasks.iterator().next()));
assertEquals(2, reader.numDocs());
reader.close();
}
private Path getOutDir() {
return new Path(_root, "out");
}
private Path getInDir() {
return new Path(_root, "in");
}
private void dump(Path path, Configuration conf) throws IOException {
FileSystem fileSystem = path.getFileSystem(conf);
System.out.println(path);
if (!fileSystem.isFile(path)) {
FileStatus[] listStatus = fileSystem.listStatus(path);
for (FileStatus fileStatus : listStatus) {
dump(fileStatus.getPath(), conf);
}
}
}
private Collection<Path> getCommitedTasks(Path path) throws IOException {
Collection<Path> result = new TreeSet<Path>();
FileSystem fileSystem = path.getFileSystem(_conf);
FileStatus[] listStatus = fileSystem.listStatus(path);
for (FileStatus fileStatus : listStatus) {
Path p = fileStatus.getPath();
if (fileStatus.isDir() && p.getName().endsWith(".commit")) {
result.add(p);
}
}
return result;
}
@Test
public void testBlurOutputFormatOverFlowTest() throws IOException, InterruptedException, ClassNotFoundException {
Path input = getInDir();
Path output = getOutDir();
_fileSystem.delete(input, true);
_fileSystem.delete(output, true);
// 1500 * 50 = 75,000
writeRecordsFile(new Path(input, "part1"), 1, 50, 1, 1500, "cf1");
// 100 * 50 = 5,000
writeRecordsFile(new Path(input, "part2"), 1, 50, 2000, 100, "cf1");
Job job = Job.getInstance(_conf, "blur index");
job.setJarByClass(BlurOutputFormatTest.class);
job.setMapperClass(CsvBlurMapper.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, input);
CsvBlurMapper.addColumns(job, "cf1", "col");
Path tablePath = new Path(new Path(_root, "table"), "test");
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setShardCount(1);
tableDescriptor.setTableUri(tablePath.toString());
tableDescriptor.setName("test");
createShardDirectories(tablePath, 1);
BlurOutputFormat.setupJob(job, tableDescriptor);
BlurOutputFormat.setOutputPath(job, output);
BlurOutputFormat.setIndexLocally(job, true);
BlurOutputFormat.setOptimizeInFlight(job, false);
assertTrue(job.waitForCompletion(true));
Counters ctrs = job.getCounters();
System.out.println("Counters: " + ctrs);
Path path = new Path(output, ShardUtil.getShardName(0));
Collection<Path> commitedTasks = getCommitedTasks(path);
assertEquals(1, commitedTasks.size());
DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(_conf, commitedTasks.iterator().next()));
assertEquals(80000, reader.numDocs());
reader.close();
}
@Test
public void testBlurOutputFormatOverFlowMultipleReducersTest() throws IOException, InterruptedException,
ClassNotFoundException {
Path input = getInDir();
Path output = getOutDir();
_fileSystem.delete(input, true);
_fileSystem.delete(output, true);
// 1500 * 50 = 75,000
writeRecordsFile(new Path(input, "part1"), 1, 50, 1, 1500, "cf1");
// 100 * 50 = 5,000
writeRecordsFile(new Path(input, "part2"), 1, 50, 2000, 100, "cf1");
Job job = Job.getInstance(_conf, "blur index");
job.setJarByClass(BlurOutputFormatTest.class);
job.setMapperClass(CsvBlurMapper.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, input);
CsvBlurMapper.addColumns(job, "cf1", "col");
Path tablePath = new Path(new Path(_root, "table"), "test");
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setShardCount(2);
tableDescriptor.setTableUri(tablePath.toString());
tableDescriptor.setName("test");
createShardDirectories(output, 2);
BlurOutputFormat.setupJob(job, tableDescriptor);
BlurOutputFormat.setOutputPath(job, output);
BlurOutputFormat.setIndexLocally(job, false);
BlurOutputFormat.setDocumentBufferStrategy(job, DocumentBufferStrategyHeapSize.class);
BlurOutputFormat.setMaxDocumentBufferHeapSize(job, 128 * 1024);
assertTrue(job.waitForCompletion(true));
Counters ctrs = job.getCounters();
System.out.println("Counters: " + ctrs);
long total = 0;
for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
Path path = new Path(output, ShardUtil.getShardName(i));
Collection<Path> commitedTasks = getCommitedTasks(path);
assertEquals(1, commitedTasks.size());
DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(_conf, commitedTasks.iterator().next()));
total += reader.numDocs();
reader.close();
}
assertEquals(80000, total);
}
@Test
public void testBlurOutputFormatOverFlowMultipleReducersWithReduceMultiplierTest() throws IOException,
InterruptedException, ClassNotFoundException {
Path input = getInDir();
Path output = getOutDir();
_fileSystem.delete(input, true);
_fileSystem.delete(output, true);
// 1500 * 50 = 75,000
writeRecordsFile(new Path(input, "part1"), 1, 50, 1, 1500, "cf1");
// 100 * 50 = 5,000
writeRecordsFile(new Path(input, "part2"), 1, 50, 2000, 100, "cf1");
Job job = Job.getInstance(_conf, "blur index");
job.setJarByClass(BlurOutputFormatTest.class);
job.setMapperClass(CsvBlurMapper.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, input);
CsvBlurMapper.addColumns(job, "cf1", "col");
Path tablePath = new Path(new Path(_root, "table"), "test");
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setShardCount(7);
tableDescriptor.setTableUri(tablePath.toString());
tableDescriptor.setName("test");
createShardDirectories(output, 7);
BlurOutputFormat.setupJob(job, tableDescriptor);
BlurOutputFormat.setOutputPath(job, output);
int multiple = 2;
BlurOutputFormat.setReducerMultiplier(job, multiple);
assertTrue(job.waitForCompletion(true));
Counters ctrs = job.getCounters();
System.out.println("Counters: " + ctrs);
long total = 0;
for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
Path path = new Path(output, ShardUtil.getShardName(i));
Collection<Path> commitedTasks = getCommitedTasks(path);
assertTrue(commitedTasks.size() >= multiple);
for (Path p : commitedTasks) {
DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(_conf, p));
total += reader.numDocs();
reader.close();
}
}
assertEquals(80000, total);
}
@Test(expected = IllegalArgumentException.class)
public void testBlurOutputFormatValidateReducerCount() throws IOException, InterruptedException,
ClassNotFoundException {
Path input = getInDir();
Path output = getOutDir();
_fileSystem.delete(input, true);
_fileSystem.delete(output, true);
writeRecordsFile(new Path(input, "part1"), 1, 1, 1, 1, "cf1");
writeRecordsFile(new Path(input, "part2"), 1, 1, 2, 1, "cf1");
Job job = Job.getInstance(_conf, "blur index");
job.setJarByClass(BlurOutputFormatTest.class);
job.setMapperClass(CsvBlurMapper.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, input);
CsvBlurMapper.addColumns(job, "cf1", "col");
Path tablePath = new Path(new Path(_root, "table"), "test");
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setShardCount(1);
tableDescriptor.setTableUri(tablePath.toString());
tableDescriptor.setName("test");
createShardDirectories(getOutDir(), 1);
BlurOutputFormat.setupJob(job, tableDescriptor);
BlurOutputFormat.setOutputPath(job, output);
BlurOutputFormat.setReducerMultiplier(job, 2);
job.setNumReduceTasks(4);
job.submit();
}
// @TODO this test to fail sometimes due to issues in the MR MiniCluster
// @Test
public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException,
ClassNotFoundException {
Path input = getInDir();
Path output = getOutDir();
_fileSystem.delete(input, true);
_fileSystem.delete(output, true);
// 1500 * 50 = 75,000
writeRecordsFile(new Path(input, "part1"), 1, 50, 1, 1500, "cf1");
// 100 * 5000 = 500,000
writeRecordsFile(new Path(input, "part2"), 1, 5000, 2000, 100, "cf1");
Job job = Job.getInstance(_conf, "blur index");
job.setJarByClass(BlurOutputFormatTest.class);
job.setMapperClass(CsvBlurMapper.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, input);
CsvBlurMapper.addColumns(job, "cf1", "col");
Path tablePath = new Path(new Path(_root, "table"), "test");
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setShardCount(2);
tableDescriptor.setTableUri(tablePath.toString());
tableDescriptor.setName("test");
createShardDirectories(getOutDir(), 2);
BlurOutputFormat.setupJob(job, tableDescriptor);
BlurOutputFormat.setOutputPath(job, output);
BlurOutputFormat.setIndexLocally(job, false);
job.submit();
boolean killCalled = false;
while (!job.isComplete()) {
Thread.sleep(1000);
System.out.printf("Killed [" + killCalled + "] Map [%f] Reduce [%f]%n", job.mapProgress() * 100,
job.reduceProgress() * 100);
if (job.reduceProgress() > 0.7 && !killCalled) {
job.killJob();
killCalled = true;
}
}
assertFalse(job.isSuccessful());
for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
Path path = new Path(output, ShardUtil.getShardName(i));
FileSystem fileSystem = path.getFileSystem(job.getConfiguration());
FileStatus[] listStatus = fileSystem.listStatus(path);
assertEquals(toString(listStatus), 0, listStatus.length);
}
}
private String toString(FileStatus[] listStatus) {
if (listStatus == null || listStatus.length == 0) {
return "";
}
String s = "";
for (FileStatus fileStatus : listStatus) {
if (s.length() > 0) {
s += ",";
}
s += fileStatus.getPath();
}
return s;
}
public static String readFile(Path file) throws IOException {
DataInputStream f = _fileSystem.open(file);
BufferedReader b = new BufferedReader(new InputStreamReader(f));
StringBuilder result = new StringBuilder();
String line = b.readLine();
while (line != null) {
result.append(line);
result.append('\n');
line = b.readLine();
}
b.close();
return result.toString();
}
private Path writeRecordsFile(Path file, int starintgRowId, int numberOfRows, int startRecordId, int numberOfRecords,
String family) throws IOException {
_fileSystem.delete(file, false);
DataOutputStream f = _fileSystem.create(file);
PrintWriter writer = new PrintWriter(f);
for (int row = 0; row < numberOfRows; row++) {
for (int record = 0; record < numberOfRecords; record++) {
writer.println(getRecord(row + starintgRowId, record + startRecordId, family));
}
}
writer.close();
return file;
}
private void createShardDirectories(Path outDir, int shardCount) throws IOException {
_fileSystem.mkdirs(outDir);
for (int i = 0; i < shardCount; i++) {
_fileSystem.mkdirs(new Path(outDir, ShardUtil.getShardName(i)));
}
}
private String getRecord(int rowId, int recordId, String family) {
return rowId + "," + recordId + "," + family + ",valuetoindex";
}
}