/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.zebra;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configured;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.Tuple;
public class BaseTestCase extends Configured {
protected static PigServer pigServer = null;
protected static Configuration conf = null;
protected static FileSystem fs = null;
protected static String zebraJar = null;
// by default, we use local model for testing
protected static TestMode mode = TestMode.local;
protected enum TestMode {
local, cluster
};
/*
* Initialized for the test case;
*/
public static void init() throws Exception {
if (conf == null) {
conf = new Configuration();
if (System.getProperty("mapred.job.queue.name") != null) {
//conf.set("mapred.job.queue.name", "grideng");
conf.set("mapred.job.queue.name", System.getProperty("mapred.job.queue.name"));
}
}
if (System.getenv("hadoop.log.dir") == null) {
String base = new File(".").getPath(); // getAbsolutePath();
System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
}
String str = System.getenv("ZebraTestMode");
if (str != null && str.equals("cluster")) {
mode = TestMode.cluster;
}
if (mode == TestMode.cluster) {
System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
System.out.println(" get env user name: " + System.getenv("USER"));
if (System.getenv("HADOOP_HOME") == null) {
System.out.println("Please set HADOOP_HOME for cluster testing mode");
System.exit(0);
}
if (System.getenv("USER") == null) {
System.out.println("Please set USER for cluster testing mode");
System.exit(0);
}
String zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
File file = new File(zebraJar);
if (!file.exists()) {
System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
System.exit(0);
}
pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil.toProperties(conf));
pigServer.registerJar(zebraJar);
Path path = new Path("/user/" + System.getenv("USER"));
fs = path.getFileSystem(conf);
} else {
pigServer = new PigServer(ExecType.LOCAL);
fs = LocalFileSystem.get(conf);
}
}
protected static void removeDir(Path path) throws IOException {
String command = null;
if (mode == TestMode.cluster) {
command = System.getenv("HADOOP_HOME") +"/bin/hadoop fs -rmr " + path.toString();
} else{
command = "rm -rf " + path.toString();
}
Runtime runtime = Runtime.getRuntime();
Process proc = runtime.exec(command);
try {
proc.waitFor();
} catch (InterruptedException e) {
System.err.println(e);
}
}
protected static Path getTableFullPath(String tableName) {
if (mode == TestMode.cluster) {
return new Path("/user/" + System.getenv("USER") + "/" + tableName);
} else {
return new Path(fs.getWorkingDirectory().toString().split(":")[1] + "/" + tableName);
}
}
/**
* Verify union output table with expected results
*
*/
protected int verifyTable(HashMap<Integer, ArrayList<ArrayList<Object>>> resultTable,
int keyColumn, int tblIdxCol, Iterator<Tuple> it) throws IOException {
int numbRows = 0;
int index = 0, rowIndex = -1, rowCount = -1, prevIndex = -1;
Object value;
boolean first = true;
ArrayList<ArrayList<Object>> rows = null;
while (it.hasNext()) {
Tuple rowValues = it.next();
if (first) {
index = (Integer) rowValues.get(tblIdxCol);
Assert.assertNotSame(prevIndex, index);
rows = resultTable.get(index);
rowIndex = 0;
rowCount = rows.size();
first = false;
}
value = rows.get(rowIndex++).get(keyColumn);
Assert.assertEquals("Table comparison error for row : " + numbRows + " - no key found for : "
+ rowValues.get(keyColumn), value, rowValues.get(keyColumn));
if (rowIndex == rowCount)
{
// current table is run out; start on a new table for next iteration
first = true;
prevIndex = index;
}
++numbRows;
}
return numbRows;
}
public static void checkTableExists(boolean expected, String strDir) throws IOException {
File theDir = null;
boolean actual = false;
theDir = new File(strDir.split(":")[0]);
actual = fs.exists(new Path (theDir.toString()));
System.out.println("the dir : "+ theDir.toString());
if (actual != expected){
Assert.fail("dir exists or not is different from what expected.");
}
}
}