package org.apache.blur.mapreduce.lib;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import org.apache.blur.MiniCluster;
import org.apache.blur.server.TableContext;
import org.apache.blur.store.buffer.BufferStore;
import org.apache.blur.thirdparty.thrift_0_9_0.TException;
import org.apache.blur.thrift.BlurClient;
import org.apache.blur.thrift.generated.Blur.Iface;
import org.apache.blur.thrift.generated.BlurException;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.blur.thrift.generated.TableStats;
import org.apache.blur.utils.GCWatcher;
import org.apache.blur.utils.JavaHome;
import org.apache.blur.utils.ShardUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
public class BlurOutputFormatMiniClusterTest {
private static Configuration conf = new Configuration();
private static FileSystem fileSystem;
private static Path TEST_ROOT_DIR;
private static MiniCluster miniCluster;
private Path inDir = new Path(TEST_ROOT_DIR + "/in");
private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir",
"./target/tmp_BlurOutputFormatMiniClusterTest"));
@BeforeClass
public static void setupTest() throws Exception {
GCWatcher.init(0.60);
JavaHome.checkJavaHome();
LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
testDirectory.mkdirs();
Path directory = new Path(testDirectory.getPath());
FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
FsAction userAction = dirPermissions.getUserAction();
FsAction groupAction = dirPermissions.getGroupAction();
FsAction otherAction = dirPermissions.getOtherAction();
StringBuilder builder = new StringBuilder();
builder.append(userAction.ordinal());
builder.append(groupAction.ordinal());
builder.append(otherAction.ordinal());
String dirPermissionNum = builder.toString();
System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
testDirectory.delete();
miniCluster = new MiniCluster();
miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, false);
TEST_ROOT_DIR = new Path(miniCluster.getFileSystemUri().toString() + "/blur_test");
System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log");
try {
fileSystem = TEST_ROOT_DIR.getFileSystem(conf);
} catch (IOException io) {
throw new RuntimeException("problem getting local fs", io);
}
FileSystem.setDefaultUri(conf, miniCluster.getFileSystemUri());
miniCluster.startMrMiniCluster();
conf = miniCluster.getMRConfiguration();
BufferStore.initNewBuffer(128, 128 * 128);
}
@AfterClass
public static void teardown() throws IOException {
if (miniCluster != null) {
miniCluster.stopMrMiniCluster();
}
miniCluster.shutdownBlurCluster();
rm(new File("build"));
}
private static void rm(File file) {
if (!file.exists()) {
return;
}
if (file.isDirectory()) {
for (File f : file.listFiles()) {
rm(f);
}
}
file.delete();
}
@Before
public void setup() {
TableContext.clear();
}
@Test
public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException, BlurException,
TException {
fileSystem.delete(inDir, true);
String tableName = "testBlurOutputFormat";
writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
Job job = Job.getInstance(conf, "blur index");
job.setJarByClass(BlurOutputFormatMiniClusterTest.class);
job.setMapperClass(CsvBlurMapper.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
String tableUri = new Path(TEST_ROOT_DIR + "/blur/" + tableName).makeQualified(fileSystem.getUri(),
fileSystem.getWorkingDirectory()).toString();
CsvBlurMapper.addColumns(job, "cf1", "col");
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setShardCount(1);
tableDescriptor.setTableUri(tableUri);
tableDescriptor.setName(tableName);
Iface client = getClient();
client.createTable(tableDescriptor);
BlurOutputFormat.setupJob(job, tableDescriptor);
Path output = new Path(TEST_ROOT_DIR + "/out");
BlurOutputFormat.setOutputPath(job, output);
Path tablePath = new Path(tableUri);
Path shardPath = new Path(tablePath, ShardUtil.getShardName(0));
FileStatus[] listStatus = fileSystem.listStatus(shardPath);
System.out.println("======" + listStatus.length);
for (FileStatus fileStatus : listStatus) {
System.out.println(fileStatus.getPath());
}
assertEquals(3, listStatus.length);
assertTrue(job.waitForCompletion(true));
Counters ctrs = job.getCounters();
System.out.println("Counters: " + ctrs);
client.loadData(tableName, output.toString());
while (true) {
TableStats tableStats = client.tableStats(tableName);
System.out.println(tableStats);
if (tableStats.getRowCount() > 0) {
break;
}
Thread.sleep(100);
}
assertTrue(fileSystem.exists(tablePath));
assertFalse(fileSystem.isFile(tablePath));
FileStatus[] listStatusAfter = fileSystem.listStatus(shardPath);
assertEquals(12, listStatusAfter.length);
}
private Iface getClient() {
return BlurClient.getClient(miniCluster.getControllerConnectionStr());
}
public static String readFile(String name) throws IOException {
DataInputStream f = fileSystem.open(new Path(TEST_ROOT_DIR + "/" + name));
BufferedReader b = new BufferedReader(new InputStreamReader(f));
StringBuilder result = new StringBuilder();
String line = b.readLine();
while (line != null) {
result.append(line);
result.append('\n');
line = b.readLine();
}
b.close();
return result.toString();
}
private Path writeRecordsFile(String name, int starintgRowId, int numberOfRows, int startRecordId,
int numberOfRecords, String family) throws IOException {
// "1,1,cf1,val1"
Path file = new Path(TEST_ROOT_DIR + "/" + name);
fileSystem.delete(file, false);
DataOutputStream f = fileSystem.create(file);
PrintWriter writer = new PrintWriter(f);
for (int row = 0; row < numberOfRows; row++) {
for (int record = 0; record < numberOfRecords; record++) {
writer.println(getRecord(row + starintgRowId, record + startRecordId, family));
}
}
writer.close();
return file;
}
private String getRecord(int rowId, int recordId, String family) {
return rowId + "," + recordId + "," + family + ",valuetoindex";
}
}