/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql;
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Deque;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hive.cli.CliDriver;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.cli.control.AbstractCliConfig;
import org.apache.hadoop.hive.common.io.CachingPrintStream;
import org.apache.hadoop.hive.common.io.DigestPrintStream;
import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream;
import org.apache.hadoop.hive.common.io.SortPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.LlapItUtils;
import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.hbase.HBaseStore;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.processors.CommandProcessor;
import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.processors.HiveCommand;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.common.util.StreamPrinter;
import org.apache.logging.log4j.util.Strings;
import org.apache.tools.ant.BuildException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableList;
import junit.framework.TestSuite;
/**
* QTestUtil.
*
*/
public class QTestUtil {
public static final String UTF_8 = "UTF-8";
// security property names
private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri";
private static final String CRLF = System.getProperty("line.separator");
public static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
private static final Logger LOG = LoggerFactory.getLogger("QTestUtil");
private final static String defaultInitScript = "q_test_init.sql";
private final static String defaultCleanupScript = "q_test_cleanup.sql";
private final String[] testOnlyCommands = new String[]{"crypto"};
private static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp
private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target
private String testWarehouse;
private final String testFiles;
protected final String outDir;
protected final String logDir;
private final TreeMap<String, String> qMap;
private final Set<String> qSkipSet;
private final Set<String> qSortSet;
private final Set<String> qSortQuerySet;
private final Set<String> qHashQuerySet;
private final Set<String> qSortNHashQuerySet;
private final Set<String> qNoSessionReuseQuerySet;
private final Set<String> qJavaVersionSpecificOutput;
private static final String SORT_SUFFIX = ".sorted";
private final Set<String> srcTables;
private final Set<String> srcUDFs;
private final MiniClusterType clusterType;
private final FsType fsType;
private ParseDriver pd;
protected Hive db;
protected QueryState queryState;
protected HiveConf conf;
private Driver drv;
private BaseSemanticAnalyzer sem;
protected final boolean overWrite;
private CliDriver cliDriver;
private HadoopShims.MiniMrShim mr = null;
private HadoopShims.MiniDFSShim dfs = null;
private FileSystem fs;
private HadoopShims.HdfsEncryptionShim hes = null;
private MiniLlapCluster llapCluster = null;
private String hadoopVer = null;
private QTestSetup setup = null;
private TezSessionState tezSessionState = null;
private SparkSession sparkSession = null;
private boolean isSessionStateStarted = false;
private static final String javaVersion = getJavaVersion();
private final String initScript;
private final String cleanupScript;
private boolean useHBaseMetastore = false;
public interface SuiteAddTestFunctor {
public void addTestToSuite(TestSuite suite, Object setup, String tName);
}
private HBaseTestingUtility utility;
public static Set<String> getSrcTables() {
HashSet<String> srcTables = new HashSet<String>();
// FIXME: moved default value to here...for now
// i think this features is never really used from the command line
String defaultTestSrcTables = "src,src1,srcbucket,srcbucket2,src_json,src_thrift,src_sequencefile,srcpart,alltypesorc,src_hbase,cbo_t1,cbo_t2,cbo_t3,src_cbo,part,lineitem";
for (String srcTable : System.getProperty("test.src.tables", defaultTestSrcTables).trim().split(",")) {
srcTable = srcTable.trim();
if (!srcTable.isEmpty()) {
srcTables.add(srcTable);
}
}
if (srcTables.isEmpty()) {
throw new RuntimeException("Source tables cannot be empty");
}
return srcTables;
}
/**
* Returns the default UDF names which should not be removed when resetting the test database
* @return The list of the UDF names not to remove
*/
private Set<String> getSrcUDFs() {
HashSet<String> srcUDFs = new HashSet<String>();
// FIXME: moved default value to here...for now
// i think this features is never really used from the command line
String defaultTestSrcUDFs = "qtest_get_java_boolean";
for (String srcUDF : System.getProperty("test.src.udfs", defaultTestSrcUDFs).trim().split(","))
{
srcUDF = srcUDF.trim();
if (!srcUDF.isEmpty()) {
srcUDFs.add(srcUDF);
}
}
if (srcUDFs.isEmpty()) {
throw new RuntimeException("Source UDFs cannot be empty");
}
return srcUDFs;
}
public HiveConf getConf() {
return conf;
}
public boolean deleteDirectory(File path) {
if (path.exists()) {
File[] files = path.listFiles();
for (File file : files) {
if (file.isDirectory()) {
deleteDirectory(file);
} else {
file.delete();
}
}
}
return (path.delete());
}
public void copyDirectoryToLocal(Path src, Path dest) throws Exception {
FileSystem srcFs = src.getFileSystem(conf);
FileSystem destFs = dest.getFileSystem(conf);
if (srcFs.exists(src)) {
FileStatus[] files = srcFs.listStatus(src);
for (FileStatus file : files) {
String name = file.getPath().getName();
Path dfs_path = file.getPath();
Path local_path = new Path(dest, name);
// If this is a source table we do not copy it out
if (srcTables.contains(name)) {
continue;
}
if (file.isDirectory()) {
if (!destFs.exists(local_path)) {
destFs.mkdirs(local_path);
}
copyDirectoryToLocal(dfs_path, local_path);
} else {
srcFs.copyToLocalFile(dfs_path, local_path);
}
}
}
}
static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)");
static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)");
public void normalizeNames(File path) throws Exception {
if (path.isDirectory()) {
File[] files = path.listFiles();
for (File file : files) {
normalizeNames(file);
}
} else {
Matcher m = reduceTok.matcher(path.getName());
if (m.matches()) {
String name = m.group(1) + "reduce" + m.group(3);
path.renameTo(new File(path.getParent(), name));
} else {
m = mapTok.matcher(path.getName());
if (m.matches()) {
String name = m.group(1) + "map_" + m.group(3);
path.renameTo(new File(path.getParent(), name));
}
}
}
}
public String getOutputDirectory() {
return outDir;
}
public String getLogDirectory() {
return logDir;
}
private String getHadoopMainVersion(String input) {
if (input == null) {
return null;
}
Pattern p = Pattern.compile("^(\\d+\\.\\d+).*");
Matcher m = p.matcher(input);
if (m.matches()) {
return m.group(1);
}
return null;
}
public void initConf() throws Exception {
String vectorizationEnabled = System.getProperty("test.vectorization.enabled");
if(vectorizationEnabled != null && vectorizationEnabled.equalsIgnoreCase("true")) {
conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, true);
}
if (!useHBaseMetastore) {
// Plug verifying metastore in for testing DirectSQL.
conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
"org.apache.hadoop.hive.metastore.VerifyingObjectStore");
} else {
conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName());
conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true);
}
if (mr != null) {
mr.setupConfiguration(conf);
// TODO Ideally this should be done independent of whether mr is setup or not.
setFsRelatedProperties(conf, fs.getScheme().equals("file"),fs);
}
if (llapCluster != null) {
Configuration clusterSpecificConf = llapCluster.getClusterSpecificConfiguration();
for (Map.Entry<String, String> confEntry : clusterSpecificConf) {
// Conf.get takes care of parameter replacement, iterator.value does not.
conf.set(confEntry.getKey(), clusterSpecificConf.get(confEntry.getKey()));
}
}
}
private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem fs) {
String fsUriString = fs.getUri().toString();
// Different paths if running locally vs a remote fileSystem. Ideally this difference should not exist.
Path warehousePath;
Path jarPath;
Path userInstallPath;
if (isLocalFs) {
String buildDir = System.getProperty(BUILD_DIR_PROPERTY);
Preconditions.checkState(Strings.isNotBlank(buildDir));
Path path = new Path(fsUriString, buildDir);
// Create a fake fs root for local fs
Path localFsRoot = new Path(path, "localfs");
warehousePath = new Path(localFsRoot, "warehouse");
jarPath = new Path(localFsRoot, "jar");
userInstallPath = new Path(localFsRoot, "user_install");
} else {
// TODO Why is this changed from the default in hive-conf?
warehousePath = new Path(fsUriString, "/build/ql/test/data/warehouse/");
jarPath = new Path(new Path(fsUriString, "/user"), "hive");
userInstallPath = new Path(fsUriString, "/user");
}
warehousePath = fs.makeQualified(warehousePath);
jarPath = fs.makeQualified(jarPath);
userInstallPath = fs.makeQualified(userInstallPath);
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsUriString);
// Remote dirs
conf.setVar(ConfVars.METASTOREWAREHOUSE, warehousePath.toString());
conf.setVar(ConfVars.HIVE_JAR_DIRECTORY, jarPath.toString());
conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, userInstallPath.toString());
// ConfVars.SCRATCHDIR - {test.tmp.dir}/scratchdir
// Local dirs
// ConfVars.LOCALSCRATCHDIR - {test.tmp.dir}/localscratchdir
// TODO Make sure to cleanup created dirs.
}
private void createRemoteDirs() {
assert fs != null;
Path warehousePath = fs.makeQualified(new Path(conf.getVar(ConfVars.METASTOREWAREHOUSE)));
assert warehousePath != null;
Path hiveJarPath = fs.makeQualified(new Path(conf.getVar(ConfVars.HIVE_JAR_DIRECTORY)));
assert hiveJarPath != null;
Path userInstallPath = fs.makeQualified(new Path(conf.getVar(ConfVars.HIVE_USER_INSTALL_DIR)));
assert userInstallPath != null;
try {
fs.mkdirs(warehousePath);
} catch (IOException e) {
LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
e.getMessage());
}
try {
fs.mkdirs(hiveJarPath);
} catch (IOException e) {
LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
e.getMessage());
}
try {
fs.mkdirs(userInstallPath);
} catch (IOException e) {
LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
e.getMessage());
}
}
private enum CoreClusterType {
MR,
TEZ,
SPARK
}
public enum FsType {
local,
hdfs,
encrypted_hdfs,
}
public enum MiniClusterType {
mr(CoreClusterType.MR, FsType.hdfs),
tez(CoreClusterType.TEZ, FsType.hdfs),
tez_local(CoreClusterType.TEZ, FsType.local),
spark(CoreClusterType.SPARK, FsType.local),
miniSparkOnYarn(CoreClusterType.SPARK, FsType.hdfs),
llap(CoreClusterType.TEZ, FsType.hdfs),
llap_local(CoreClusterType.TEZ, FsType.local),
none(CoreClusterType.MR, FsType.local);
private final CoreClusterType coreClusterType;
private final FsType defaultFsType;
MiniClusterType(CoreClusterType coreClusterType, FsType defaultFsType) {
this.coreClusterType = coreClusterType;
this.defaultFsType = defaultFsType;
}
public CoreClusterType getCoreClusterType() {
return coreClusterType;
}
public FsType getDefaultFsType() {
return defaultFsType;
}
public static MiniClusterType valueForString(String type) {
// Replace this with valueOf.
if (type.equals("miniMR")) {
return mr;
} else if (type.equals("tez")) {
return tez;
} else if (type.equals("tez_local")) {
return tez_local;
} else if (type.equals("spark")) {
return spark;
} else if (type.equals("miniSparkOnYarn")) {
return miniSparkOnYarn;
} else if (type.equals("llap")) {
return llap;
} else if (type.equals("llap_local")) {
return llap_local;
} else {
return none;
}
}
}
private String getKeyProviderURI() {
// Use the target directory if it is not specified
String HIVE_ROOT = AbstractCliConfig.HIVE_ROOT;
String keyDir = HIVE_ROOT + "ql/target/";
// put the jks file in the current test path only for test purpose
return "jceks://file" + new Path(keyDir, "test.jks").toUri();
}
private void startMiniHBaseCluster() throws Exception {
Configuration hbaseConf = HBaseConfiguration.create();
hbaseConf.setInt("hbase.master.info.port", -1);
utility = new HBaseTestingUtility(hbaseConf);
utility.startMiniCluster();
conf = new HiveConf(utility.getConfiguration(), Driver.class);
HBaseAdmin admin = utility.getHBaseAdmin();
// Need to use reflection here to make compilation pass since HBaseIntegrationTests
// is not compiled in hadoop-1. All HBaseMetastore tests run under hadoop-2, so this
// guarantee HBaseIntegrationTests exist when we hitting this code path
java.lang.reflect.Method initHBaseMetastoreMethod = Class.forName(
"org.apache.hadoop.hive.metastore.hbase.HBaseStoreTestUtil")
.getMethod("initHBaseMetastore", HBaseAdmin.class, HiveConf.class);
initHBaseMetastoreMethod.invoke(null, admin, conf);
conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName());
conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true);
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadoopVer, String initScript, String cleanupScript,
boolean useHBaseMetastore, boolean withLlapIo) throws Exception {
this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
useHBaseMetastore, withLlapIo, null);
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadzoopVer, String initScript, String cleanupScript,
boolean useHBaseMetastore, boolean withLlapIo, FsType fsType)
throws Exception {
LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," +
" hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," +
" fsType={}"
, outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
useHBaseMetastore, withLlapIo, fsType);
Preconditions.checkNotNull(clusterType, "ClusterType cannot be null");
if (fsType != null) {
this.fsType = fsType;
} else {
this.fsType = clusterType.getDefaultFsType();
}
this.outDir = outDir;
this.logDir = logDir;
this.useHBaseMetastore = useHBaseMetastore;
this.srcTables=getSrcTables();
this.srcUDFs = getSrcUDFs();
// HIVE-14443 move this fall-back logic to CliConfigs
if (confDir != null && !confDir.isEmpty()) {
HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
}
queryState = new QueryState.Builder().withHiveConf(new HiveConf(Driver.class)).build();
if (useHBaseMetastore) {
startMiniHBaseCluster();
} else {
conf = queryState.getConf();
}
this.hadoopVer = getHadoopMainVersion(hadoopVer);
qMap = new TreeMap<String, String>();
qSkipSet = new HashSet<String>();
qSortSet = new HashSet<String>();
qSortQuerySet = new HashSet<String>();
qHashQuerySet = new HashSet<String>();
qSortNHashQuerySet = new HashSet<String>();
qNoSessionReuseQuerySet = new HashSet<String>();
qJavaVersionSpecificOutput = new HashSet<String>();
this.clusterType = clusterType;
HadoopShims shims = ShimLoader.getHadoopShims();
setupFileSystem(shims);
setup = new QTestSetup();
setup.preTest(conf);
setupMiniCluster(shims, confDir);
initConf();
if (withLlapIo && (clusterType == MiniClusterType.none)) {
LOG.info("initializing llap IO");
LlapProxy.initializeLlapIo(conf);
}
// Use the current directory if it is not specified
String dataDir = conf.get("test.data.files");
if (dataDir == null) {
dataDir = new File(".").getAbsolutePath() + "/data/files";
}
testFiles = dataDir;
// Use the current directory if it is not specified
String scriptsDir = conf.get("test.data.scripts");
if (scriptsDir == null) {
scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
}
this.initScript = scriptsDir + File.separator + initScript;
this.cleanupScript = scriptsDir + File.separator + cleanupScript;
overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));
init();
}
private void setupFileSystem(HadoopShims shims) throws IOException {
if (fsType == FsType.local) {
fs = FileSystem.getLocal(conf);
} else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) {
int numDataNodes = 4;
if (fsType == FsType.encrypted_hdfs) {
// Set the security key provider so that the MiniDFS cluster is initialized
// with encryption
conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI());
conf.setInt("fs.trash.interval", 50);
dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
fs = dfs.getFileSystem();
// set up the java key provider for encrypted hdfs cluster
hes = shims.createHdfsEncryptionShim(fs, conf);
LOG.info("key provider is initialized");
} else {
dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
fs = dfs.getFileSystem();
}
} else {
throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]");
}
}
private void setupMiniCluster(HadoopShims shims, String confDir) throws
IOException {
String uriString = fs.getUri().toString();
if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
if (confDir != null && !confDir.isEmpty()) {
conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ "/tez-site.xml"));
}
int numTrackers = 2;
if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) {
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir);
} else {
}
if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) {
mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local);
} else {
mr = shims.getMiniTezCluster(conf, numTrackers, uriString,
EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType));
}
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
mr = shims.getMiniSparkCluster(conf, 2, uriString, 1);
} else if (clusterType == MiniClusterType.mr) {
mr = shims.getMiniMrCluster(conf, 2, uriString, 1);
}
}
public void shutdown() throws Exception {
if (System.getenv(QTEST_LEAVE_FILES) == null) {
cleanUp();
}
if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
SessionState.get().getTezSession().close(false);
}
setup.tearDown();
if (sparkSession != null) {
try {
SparkSessionManagerImpl.getInstance().closeSession(sparkSession);
} catch (Exception ex) {
LOG.error("Error closing spark session.", ex);
} finally {
sparkSession = null;
}
}
if (useHBaseMetastore) {
utility.shutdownMiniCluster();
}
if (mr != null) {
mr.shutdown();
mr = null;
}
FileSystem.closeAll();
if (dfs != null) {
dfs.shutdown();
dfs = null;
}
Hive.closeCurrent();
}
public String readEntireFileIntoString(File queryFile) throws IOException {
InputStreamReader isr = new InputStreamReader(
new BufferedInputStream(new FileInputStream(queryFile)), QTestUtil.UTF_8);
StringWriter sw = new StringWriter();
try {
IOUtils.copy(isr, sw);
} finally {
if (isr != null) {
isr.close();
}
}
return sw.toString();
}
public void addFile(String queryFile) throws IOException {
addFile(queryFile, false);
}
public void addFile(String queryFile, boolean partial) throws IOException {
addFile(new File(queryFile));
}
public void addFile(File qf) throws IOException {
addFile(qf, false);
}
public void addFile(File qf, boolean partial) throws IOException {
String query = readEntireFileIntoString(qf);
qMap.put(qf.getName(), query);
if (partial) return;
if(checkHadoopVersionExclude(qf.getName(), query)) {
qSkipSet.add(qf.getName());
}
if (checkNeedJavaSpecificOutput(qf.getName(), query)) {
qJavaVersionSpecificOutput.add(qf.getName());
}
if (matches(SORT_BEFORE_DIFF, query)) {
qSortSet.add(qf.getName());
} else if (matches(SORT_QUERY_RESULTS, query)) {
qSortQuerySet.add(qf.getName());
} else if (matches(HASH_QUERY_RESULTS, query)) {
qHashQuerySet.add(qf.getName());
} else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) {
qSortNHashQuerySet.add(qf.getName());
}
if (matches(NO_SESSION_REUSE, query)) {
qNoSessionReuseQuerySet.add(qf.getName());
}
}
private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF");
private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS");
private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS");
private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS");
private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE");
private boolean matches(Pattern pattern, String query) {
Matcher matcher = pattern.matcher(query);
if (matcher.find()) {
return true;
}
return false;
}
private boolean checkHadoopVersionExclude(String fileName, String query){
// Look for a hint to not run a test on some Hadoop versions
Pattern pattern = Pattern.compile("-- (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS\\((.*)\\)");
boolean excludeQuery = false;
boolean includeQuery = false;
Set<String> versionSet = new HashSet<String>();
String hadoopVer = ShimLoader.getMajorVersion();
Matcher matcher = pattern.matcher(query);
// Each qfile may include at most one INCLUDE or EXCLUDE directive.
//
// If a qfile contains an INCLUDE directive, and hadoopVer does
// not appear in the list of versions to include, then the qfile
// is skipped.
//
// If a qfile contains an EXCLUDE directive, and hadoopVer is
// listed in the list of versions to EXCLUDE, then the qfile is
// skipped.
//
// Otherwise, the qfile is included.
if (matcher.find()) {
String prefix = matcher.group(1);
if ("EX".equals(prefix)) {
excludeQuery = true;
} else {
includeQuery = true;
}
String versions = matcher.group(2);
for (String s : versions.split("\\,")) {
s = s.trim();
versionSet.add(s);
}
}
if (matcher.find()) {
//2nd match is not supposed to be there
String message = "QTestUtil: qfile " + fileName
+ " contains more than one reference to (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS";
throw new UnsupportedOperationException(message);
}
if (excludeQuery && versionSet.contains(hadoopVer)) {
System.out.println("QTestUtil: " + fileName
+ " EXCLUDE list contains Hadoop Version " + hadoopVer + ". Skipping...");
return true;
} else if (includeQuery && !versionSet.contains(hadoopVer)) {
System.out.println("QTestUtil: " + fileName
+ " INCLUDE list does not contain Hadoop Version " + hadoopVer + ". Skipping...");
return true;
}
return false;
}
private boolean checkNeedJavaSpecificOutput(String fileName, String query) {
Pattern pattern = Pattern.compile("-- JAVA_VERSION_SPECIFIC_OUTPUT");
Matcher matcher = pattern.matcher(query);
if (matcher.find()) {
System.out.println("Test is flagged to generate Java version specific " +
"output. Since we are using Java version " + javaVersion +
", we will generated Java " + javaVersion + " specific " +
"output file for query file " + fileName);
return true;
}
return false;
}
/**
* Get formatted Java version to include minor version, but
* exclude patch level.
*
* @return Java version formatted as major_version.minor_version
*/
private static String getJavaVersion() {
String version = System.getProperty("java.version");
if (version == null) {
throw new NullPointerException("No java version could be determined " +
"from system properties");
}
// "java version" system property is formatted
// major_version.minor_version.patch_level.
// Find second dot, instead of last dot, to be safe
int pos = version.indexOf('.');
pos = version.indexOf('.', pos + 1);
return version.substring(0, pos);
}
/**
* Clear out any side effects of running tests
*/
public void clearPostTestEffects() throws Exception {
setup.postTest(conf);
}
public void clearKeysCreatedInTests() {
if (hes == null) {
return;
}
try {
for (String keyAlias : hes.getKeys()) {
hes.deleteKey(keyAlias);
}
} catch (IOException e) {
LOG.error("Fail to clean the keys created in test due to the error", e);
}
}
public void clearUDFsCreatedDuringTests() throws Exception {
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
}
// Delete functions created by the tests
// It is enough to remove functions from the default database, other databases are dropped
for (String udfName : db.getFunctions(DEFAULT_DATABASE_NAME, ".*")) {
if (!srcUDFs.contains(udfName)) {
db.dropFunction(DEFAULT_DATABASE_NAME, udfName);
}
}
}
/**
* Clear out any side effects of running tests
*/
public void clearTablesCreatedDuringTests() throws Exception {
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
}
conf.set("hive.metastore.filter.hook",
"org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl");
db = Hive.get(conf);
// Delete any tables other than the source tables
// and any databases other than the default database.
for (String dbName : db.getAllDatabases()) {
SessionState.get().setCurrentDatabase(dbName);
for (String tblName : db.getAllTables()) {
if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) {
Table tblObj = null;
try {
tblObj = db.getTable(tblName);
} catch (InvalidTableException e) {
LOG.warn("Trying to drop table " + e.getTableName() + ". But it does not exist.");
continue;
}
// dropping index table can not be dropped directly. Dropping the base
// table will automatically drop all its index table
if(tblObj.isIndexTable()) {
continue;
}
db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs);
} else {
// this table is defined in srcTables, drop all indexes on it
List<Index> indexes = db.getIndexes(dbName, tblName, (short)-1);
if (indexes != null && indexes.size() > 0) {
for (Index index : indexes) {
db.dropIndex(dbName, tblName, index.getIndexName(), true, true);
}
}
}
}
if (!DEFAULT_DATABASE_NAME.equals(dbName)) {
// Drop cascade, functions dropped by cascade
db.dropDatabase(dbName, true, true, true);
}
}
// delete remaining directories for external tables (can affect stats for following tests)
try {
Path p = new Path(testWarehouse);
FileSystem fileSystem = p.getFileSystem(conf);
if (fileSystem.exists(p)) {
for (FileStatus status : fileSystem.listStatus(p)) {
if (status.isDirectory() && !srcTables.contains(status.getPath().getName())) {
fileSystem.delete(status.getPath(), true);
}
}
}
} catch (IllegalArgumentException e) {
// ignore.. provides invalid url sometimes intentionally
}
SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
List<String> roleNames = db.getAllRoleNames();
for (String roleName : roleNames) {
if (!"PUBLIC".equalsIgnoreCase(roleName) && !"ADMIN".equalsIgnoreCase(roleName)) {
db.dropRole(roleName);
}
}
}
/**
* Clear out any side effects of running tests
*/
public void clearTestSideEffects() throws Exception {
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
}
// allocate and initialize a new conf since a test can
// modify conf by using 'set' commands
conf = new HiveConf(Driver.class);
initConf();
initConfFromSetup();
// renew the metastore since the cluster type is unencrypted
db = Hive.get(conf); // propagate new conf to meta store
clearTablesCreatedDuringTests();
clearUDFsCreatedDuringTests();
clearKeysCreatedInTests();
}
protected void initConfFromSetup() throws Exception {
setup.preTest(conf);
}
public void cleanUp() throws Exception {
cleanUp(null);
}
public void cleanUp(String tname) throws Exception {
boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
if(!isSessionStateStarted) {
startSessionState(canReuseSession);
}
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
}
clearTablesCreatedDuringTests();
clearUDFsCreatedDuringTests();
clearKeysCreatedInTests();
File cleanupFile = new File(cleanupScript);
if (cleanupFile.isFile()) {
String cleanupCommands = readEntireFileIntoString(cleanupFile);
LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
if(cliDriver == null) {
cliDriver = new CliDriver();
}
SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
int result = cliDriver.processLine(cleanupCommands);
if (result != 0) {
LOG.error("Failed during cleanup processLine with code={}. Ignoring", result);
// TODO Convert this to an Assert.fail once HIVE-14682 is fixed
}
SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);
} else {
LOG.info("No cleanup script detected. Skipping.");
}
// delete any contents in the warehouse dir
Path p = new Path(testWarehouse);
FileSystem fs = p.getFileSystem(conf);
try {
FileStatus [] ls = fs.listStatus(p);
for (int i=0; (ls != null) && (i<ls.length); i++) {
fs.delete(ls[i].getPath(), true);
}
} catch (FileNotFoundException e) {
// Best effort
}
// TODO: Clean up all the other paths that are created.
FunctionRegistry.unregisterTemporaryUDF("test_udaf");
FunctionRegistry.unregisterTemporaryUDF("test_error");
}
protected void runCreateTableCmd(String createTableCmd) throws Exception {
int ecode = 0;
ecode = drv.run(createTableCmd).getResponseCode();
if (ecode != 0) {
throw new Exception("create table command: " + createTableCmd
+ " failed with exit code= " + ecode);
}
return;
}
protected void runCmd(String cmd) throws Exception {
int ecode = 0;
ecode = drv.run(cmd).getResponseCode();
drv.close();
if (ecode != 0) {
throw new Exception("command: " + cmd
+ " failed with exit code= " + ecode);
}
return;
}
public void createSources() throws Exception {
createSources(null);
}
public void createSources(String tname) throws Exception {
boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
if(!isSessionStateStarted) {
startSessionState(canReuseSession);
}
if(cliDriver == null) {
cliDriver = new CliDriver();
}
cliDriver.processLine("set test.data.dir=" + testFiles + ";");
File scriptFile = new File(this.initScript);
if (!scriptFile.isFile()) {
LOG.info("No init script detected. Skipping");
return;
}
conf.setBoolean("hive.test.init.phase", true);
String initCommands = readEntireFileIntoString(scriptFile);
LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
int result = cliDriver.processLine(initCommands);
LOG.info("Result from cliDrriver.processLine in createSources=" + result);
if (result != 0) {
Assert.fail("Failed during createSources processLine with code=" + result);
}
conf.setBoolean("hive.test.init.phase", false);
}
public void init() throws Exception {
// Create remote dirs once.
if (mr != null) {
createRemoteDirs();
}
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
String execEngine = conf.get("hive.execution.engine");
conf.set("hive.execution.engine", "mr");
SessionState.start(conf);
conf.set("hive.execution.engine", execEngine);
db = Hive.get(conf);
drv = new Driver(conf);
drv.init();
pd = new ParseDriver();
sem = new SemanticAnalyzer(queryState);
}
public void init(String tname) throws Exception {
cleanUp(tname);
createSources(tname);
cliDriver.processCmd("set hive.cli.print.header=true;");
}
public void cliInit(String tname) throws Exception {
cliInit(tname, true);
}
public String cliInit(String tname, boolean recreate) throws Exception {
if (recreate) {
cleanUp(tname);
createSources(tname);
}
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
Utilities.clearWorkMap(conf);
CliSessionState ss = createSessionState();
assert ss != null;
ss.in = System.in;
String outFileExtension = getOutFileExtension(tname);
String stdoutName = null;
if (outDir != null) {
// TODO: why is this needed?
File qf = new File(outDir, tname);
stdoutName = qf.getName().concat(outFileExtension);
} else {
stdoutName = tname + outFileExtension;
}
File outf = new File(logDir, stdoutName);
OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
if (qSortQuerySet.contains(tname)) {
ss.out = new SortPrintStream(fo, "UTF-8");
} else if (qHashQuerySet.contains(tname)) {
ss.out = new DigestPrintStream(fo, "UTF-8");
} else if (qSortNHashQuerySet.contains(tname)) {
ss.out = new SortAndDigestPrintStream(fo, "UTF-8");
} else {
ss.out = new PrintStream(fo, true, "UTF-8");
}
ss.err = new CachingPrintStream(fo, true, "UTF-8");
ss.setIsSilent(true);
SessionState oldSs = SessionState.get();
boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname);
if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
// Copy the tezSessionState from the old CliSessionState.
tezSessionState = oldSs.getTezSession();
oldSs.setTezSession(null);
ss.setTezSession(tezSessionState);
oldSs.close();
}
if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
oldSs.close();
}
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
oldSs.out.close();
}
SessionState.start(ss);
cliDriver = new CliDriver();
if (tname.equals("init_file.q")) {
ss.initFiles.add(AbstractCliConfig.HIVE_ROOT + "/data/scripts/test_init_file.sql");
}
cliDriver.processInitFiles(ss);
return outf.getAbsolutePath();
}
private CliSessionState createSessionState() {
return new CliSessionState(conf) {
@Override
public void setSparkSession(SparkSession sparkSession) {
super.setSparkSession(sparkSession);
if (sparkSession != null) {
try {
// Wait a little for cluster to init, at most 4 minutes
long endTime = System.currentTimeMillis() + 240000;
while (sparkSession.getMemoryAndCores().getSecond() <= 1) {
if (System.currentTimeMillis() >= endTime) {
String msg = "Timed out waiting for Spark cluster to init";
throw new IllegalStateException(msg);
}
Thread.sleep(100);
}
} catch (Exception e) {
String msg = "Error trying to obtain executor info: " + e;
LOG.error(msg, e);
throw new IllegalStateException(msg, e);
}
}
}
};
}
private CliSessionState startSessionState(boolean canReuseSession)
throws IOException {
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
String execEngine = conf.get("hive.execution.engine");
conf.set("hive.execution.engine", "mr");
CliSessionState ss = createSessionState();
assert ss != null;
ss.in = System.in;
ss.out = System.out;
ss.err = System.out;
SessionState oldSs = SessionState.get();
if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
// Copy the tezSessionState from the old CliSessionState.
tezSessionState = oldSs.getTezSession();
ss.setTezSession(tezSessionState);
oldSs.setTezSession(null);
oldSs.close();
}
if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
oldSs.close();
}
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
oldSs.out.close();
}
SessionState.start(ss);
isSessionStateStarted = true;
conf.set("hive.execution.engine", execEngine);
return ss;
}
public int executeAdhocCommand(String q) {
if (!q.contains(";")) {
return -1;
}
String q1 = q.split(";")[0] + ";";
LOG.debug("Executing " + q1);
return cliDriver.processLine(q1);
}
public int executeOne(String tname) {
String q = qMap.get(tname);
if (q.indexOf(";") == -1) {
return -1;
}
String q1 = q.substring(0, q.indexOf(";") + 1);
String qrest = q.substring(q.indexOf(";") + 1);
qMap.put(tname, qrest);
System.out.println("Executing " + q1);
return cliDriver.processLine(q1);
}
public int execute(String tname) {
try {
return drv.run(qMap.get(tname)).getResponseCode();
} catch (CommandNeedRetryException e) {
LOG.error("driver failed to run the command: " + tname + " due to the exception: ", e);
e.printStackTrace();
return -1;
}
}
public int executeClient(String tname1, String tname2) {
String commands = getCommand(tname1) + CRLF + getCommand(tname2);
return executeClientInternal(commands);
}
public int executeClient(String tname) {
return executeClientInternal(getCommand(tname));
}
private int executeClientInternal(String commands) {
List<String> cmds = CliDriver.splitSemiColon(commands);
int rc = 0;
String command = "";
for (String oneCmd : cmds) {
if (StringUtils.endsWith(oneCmd, "\\")) {
command += StringUtils.chop(oneCmd) + "\\;";
continue;
} else {
if (isHiveCommand(oneCmd)) {
command = oneCmd;
} else {
command += oneCmd;
}
}
if (StringUtils.isBlank(command)) {
continue;
}
if (isCommandUsedForTesting(command)) {
rc = executeTestCommand(command);
} else {
rc = cliDriver.processLine(command);
}
if (rc != 0 && !ignoreErrors()) {
break;
}
command = "";
}
if (rc == 0 && SessionState.get() != null) {
SessionState.get().setLastCommand(null); // reset
}
return rc;
}
/**
* This allows a .q file to continue executing after a statement runs into an error which is convenient
* if you want to use another hive cmd after the failure to sanity check the state of the system.
*/
private boolean ignoreErrors() {
return conf.getBoolVar(HiveConf.ConfVars.CLIIGNOREERRORS);
}
private boolean isHiveCommand(String command) {
String[] cmd = command.trim().split("\\s+");
if (HiveCommand.find(cmd) != null) {
return true;
} else if (HiveCommand.find(cmd, HiveCommand.ONLY_FOR_TESTING) != null) {
return true;
} else {
return false;
}
}
private int executeTestCommand(final String command) {
String commandName = command.trim().split("\\s+")[0];
String commandArgs = command.trim().substring(commandName.length());
if (commandArgs.endsWith(";")) {
commandArgs = StringUtils.chop(commandArgs);
}
//replace ${hiveconf:hive.metastore.warehouse.dir} with actual dir if existed.
//we only want the absolute path, so remove the header, such as hdfs://localhost:57145
String wareHouseDir = SessionState.get().getConf().getVar(ConfVars.METASTOREWAREHOUSE)
.replaceAll("^[a-zA-Z]+://.*?:\\d+", "");
commandArgs = commandArgs.replaceAll("\\$\\{hiveconf:hive\\.metastore\\.warehouse\\.dir\\}",
wareHouseDir);
if (SessionState.get() != null) {
SessionState.get().setLastCommand(commandName + " " + commandArgs.trim());
}
enableTestOnlyCmd(SessionState.get().getConf());
try {
CommandProcessor proc = getTestCommand(commandName);
if (proc != null) {
CommandProcessorResponse response = proc.run(commandArgs.trim());
int rc = response.getResponseCode();
if (rc != 0) {
SessionState.getConsole().printError(response.toString(), response.getException() != null ?
Throwables.getStackTraceAsString(response.getException()) : "");
}
return rc;
} else {
throw new RuntimeException("Could not get CommandProcessor for command: " + commandName);
}
} catch (Exception e) {
throw new RuntimeException("Could not execute test command", e);
}
}
private CommandProcessor getTestCommand(final String commandName) throws SQLException {
HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING);
if (testCommand == null) {
return null;
}
return CommandProcessorFactory
.getForHiveCommandInternal(new String[]{commandName}, SessionState.get().getConf(),
testCommand.isOnlyForTesting());
}
private void enableTestOnlyCmd(HiveConf conf){
StringBuilder securityCMDs = new StringBuilder(conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST));
for(String c : testOnlyCommands){
securityCMDs.append(",");
securityCMDs.append(c);
}
conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), securityCMDs.toString());
}
private boolean isCommandUsedForTesting(final String command) {
String commandName = command.trim().split("\\s+")[0];
HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING);
return testCommand != null;
}
private String getCommand(String tname) {
String commands = qMap.get(tname);
StringBuilder newCommands = new StringBuilder(commands.length());
int lastMatchEnd = 0;
Matcher commentMatcher = Pattern.compile("^--.*$", Pattern.MULTILINE).matcher(commands);
// remove the comments
while (commentMatcher.find()) {
newCommands.append(commands.substring(lastMatchEnd, commentMatcher.start()));
lastMatchEnd = commentMatcher.end();
}
newCommands.append(commands.substring(lastMatchEnd, commands.length()));
commands = newCommands.toString();
return commands;
}
public boolean shouldBeSkipped(String tname) {
return qSkipSet.contains(tname);
}
private String getOutFileExtension(String fname) {
String outFileExtension = ".out";
if (qJavaVersionSpecificOutput.contains(fname)) {
outFileExtension = ".java" + javaVersion + ".out";
}
return outFileExtension;
}
public void convertSequenceFileToTextFile() throws Exception {
// Create an instance of hive in order to create the tables
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
db = Hive.get(conf);
// Move all data from dest4_sequencefile to dest4
drv
.run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*");
// Drop dest4_sequencefile
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile",
true, true);
}
public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception {
String outFileExtension = getOutFileExtension(tname);
File qf = new File(outDir, tname);
String expf = outPath(outDir.toString(), tname.concat(outFileExtension));
File outf = null;
outf = new File(logDir);
outf = new File(outf, qf.getName().concat(outFileExtension));
FileWriter outfd = new FileWriter(outf);
if (e instanceof ParseException) {
outfd.write("Parse Error: ");
} else if (e instanceof SemanticException) {
outfd.write("Semantic Exception: \n");
} else {
throw e;
}
outfd.write(e.getMessage());
outfd.close();
QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
qSortSet.contains(qf.getName()));
if (overWrite) {
overwriteResults(outf.getPath(), expf);
return QTestProcessExecResult.createWithoutOutput(0);
}
return result;
}
public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception {
if (tree != null) {
String outFileExtension = getOutFileExtension(tname);
File parseDir = new File(outDir, "parse");
String expf = outPath(parseDir.toString(), tname.concat(outFileExtension));
File outf = null;
outf = new File(logDir);
outf = new File(outf, tname.concat(outFileExtension));
FileWriter outfd = new FileWriter(outf);
outfd.write(tree.toStringTree());
outfd.close();
QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
if (overWrite) {
overwriteResults(outf.getPath(), expf);
return QTestProcessExecResult.createWithoutOutput(0);
}
return exitVal;
} else {
throw new Exception("Parse tree is null");
}
}
/**
* Given the current configurations (e.g., hadoop version and execution mode), return
* the correct file name to compare with the current test run output.
* @param outDir The directory where the reference log files are stored.
* @param testName The test file name (terminated by ".out").
* @return The file name appended with the configuration values if it exists.
*/
public String outPath(String outDir, String testName) {
String ret = (new File(outDir, testName)).getPath();
// List of configurations. Currently the list consists of hadoop version and execution mode only
List<String> configs = new ArrayList<String>();
configs.add(this.hadoopVer);
Deque<String> stack = new LinkedList<String>();
StringBuilder sb = new StringBuilder();
sb.append(testName);
stack.push(sb.toString());
// example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17
for (String s: configs) {
sb.append('_');
sb.append(s);
stack.push(sb.toString());
}
while (stack.size() > 0) {
String fileName = stack.pop();
File f = new File(outDir, fileName);
if (f.exists()) {
ret = f.getPath();
break;
}
}
return ret;
}
private Pattern[] toPattern(String[] patternStrs) {
Pattern[] patterns = new Pattern[patternStrs.length];
for (int i = 0; i < patternStrs.length; i++) {
patterns[i] = Pattern.compile(patternStrs[i]);
}
return patterns;
}
private void maskPatterns(Pattern[] patterns, String fname) throws Exception {
String maskPattern = "#### A masked pattern was here ####";
String partialMaskPattern = "#### A PARTIAL masked pattern was here ####";
String line;
BufferedReader in;
BufferedWriter out;
File file = new File(fname);
File fileOrig = new File(fname + ".orig");
FileUtils.copyFile(file, fileOrig);
in = new BufferedReader(new InputStreamReader(new FileInputStream(fileOrig), "UTF-8"));
out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"));
boolean lastWasMasked = false;
boolean partialMaskWasMatched = false;
Matcher matcher;
while (null != (line = in.readLine())) {
if (fsType == FsType.encrypted_hdfs) {
for (Pattern pattern : partialReservedPlanMask) {
matcher = pattern.matcher(line);
if (matcher.find()) {
line = partialMaskPattern + " " + matcher.group(0);
partialMaskWasMatched = true;
break;
}
}
}
if (!partialMaskWasMatched) {
for (Pair<Pattern, String> pair : patternsWithMaskComments) {
Pattern pattern = pair.getLeft();
String maskComment = pair.getRight();
matcher = pattern.matcher(line);
if (matcher.find()) {
line = matcher.replaceAll(maskComment);
partialMaskWasMatched = true;
break;
}
}
for (Pattern pattern : patterns) {
line = pattern.matcher(line).replaceAll(maskPattern);
}
}
if (line.equals(maskPattern)) {
// We're folding multiple masked lines into one.
if (!lastWasMasked) {
out.write(line);
out.write("\n");
lastWasMasked = true;
partialMaskWasMatched = false;
}
} else {
out.write(line);
out.write("\n");
lastWasMasked = false;
partialMaskWasMatched = false;
}
}
in.close();
out.close();
}
private final Pattern[] planMask = toPattern(new String[] {
".*file:.*",
".*pfile:.*",
".*hdfs:.*",
".*/tmp/.*",
".*invalidscheme:.*",
".*lastUpdateTime.*",
".*lastAccessTime.*",
".*lastModifiedTime.*",
".*[Oo]wner.*",
".*CreateTime.*",
".*LastAccessTime.*",
".*Location.*",
".*LOCATION '.*",
".*transient_lastDdlTime.*",
".*last_modified_.*",
".*at org.*",
".*at sun.*",
".*at java.*",
".*at junit.*",
".*Caused by:.*",
".*LOCK_QUERYID:.*",
".*LOCK_TIME:.*",
".*grantTime.*",
".*[.][.][.] [0-9]* more.*",
".*job_[0-9_]*.*",
".*job_local[0-9_]*.*",
".*USING 'java -cp.*",
"^Deleted.*",
".*DagName:.*",
".*DagId:.*",
".*Input:.*/data/files/.*",
".*Output:.*/data/files/.*",
".*total number of created files now is.*",
".*.hive-staging.*",
"pk_-?[0-9]*_[0-9]*_[0-9]*",
"fk_-?[0-9]*_[0-9]*_[0-9]*",
".*at com\\.sun\\.proxy.*",
".*at com\\.jolbox.*",
".*at com\\.zaxxer.*",
"org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
"^Repair: Added partition to metastore.*"
});
private final Pattern[] partialReservedPlanMask = toPattern(new String[] {
"data/warehouse/(.*?/)+\\.hive-staging" // the directory might be db/table/partition
//TODO: add more expected test result here
});
/* This list may be modified by specific cli drivers to mask strings that change on every test */
private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
}};
private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment) {
return ImmutablePair.of(Pattern.compile(patternStr), maskComment);
}
public void addPatternWithMaskComment(String patternStr, String maskComment) {
patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
}
public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
assert(qMap.containsKey(tname));
String outFileExtension = getOutFileExtension(tname);
String outFileName = outPath(outDir, tname + outFileExtension);
File f = new File(logDir, tname + outFileExtension);
maskPatterns(planMask, f.getPath());
QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
outFileName, false,
qSortSet.contains(tname));
if (overWrite) {
overwriteResults(f.getPath(), outFileName);
return QTestProcessExecResult.createWithoutOutput(0);
}
return exitVal;
}
public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
throws Exception {
assert outputs.size() > 1;
maskPatterns(planMask, outputs.get(0));
for (int i = 1; i < outputs.size(); ++i) {
maskPatterns(planMask, outputs.get(i));
QTestProcessExecResult result = executeDiffCommand(
outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
if (result.getReturnCode() != 0) {
System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
return result;
}
}
return QTestProcessExecResult.createWithoutOutput(0);
}
private static void overwriteResults(String inFileName, String outFileName) throws Exception {
// This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
// once Hive uses JAVA 7.
System.out.println("Overwriting results " + inFileName + " to " + outFileName);
int result = executeCmd(new String[]{
"cp",
getQuotedString(inFileName),
getQuotedString(outFileName)
}).getReturnCode();
if (result != 0)
throw new IllegalStateException("Unexpected error while overwriting " +
inFileName + " with " + outFileName);
}
private static QTestProcessExecResult executeDiffCommand(String inFileName,
String outFileName,
boolean ignoreWhiteSpace,
boolean sortResults
) throws Exception {
QTestProcessExecResult result;
if (sortResults) {
// sort will try to open the output file in write mode on windows. We need to
// close it first.
SessionState ss = SessionState.get();
if (ss != null && ss.out != null && ss.out != System.out) {
ss.out.close();
}
String inSorted = inFileName + SORT_SUFFIX;
String outSorted = outFileName + SORT_SUFFIX;
sortFiles(inFileName, inSorted);
sortFiles(outFileName, outSorted);
inFileName = inSorted;
outFileName = outSorted;
}
ArrayList<String> diffCommandArgs = new ArrayList<String>();
diffCommandArgs.add("diff");
// Text file comparison
diffCommandArgs.add("-a");
// Ignore changes in the amount of white space
if (ignoreWhiteSpace) {
diffCommandArgs.add("-b");
}
// Add files to compare to the arguments list
diffCommandArgs.add(getQuotedString(inFileName));
diffCommandArgs.add(getQuotedString(outFileName));
result = executeCmd(diffCommandArgs);
if (sortResults) {
new File(inFileName).delete();
new File(outFileName).delete();
}
return result;
}
private static void sortFiles(String in, String out) throws Exception {
int result = executeCmd(new String[]{
"sort",
getQuotedString(in),
}, out, null).getReturnCode();
if (result != 0)
throw new IllegalStateException("Unexpected error while sorting " + in);
}
private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception {
return executeCmd(args, null, null);
}
private static QTestProcessExecResult executeCmd(String[] args) throws Exception {
return executeCmd(args, null, null);
}
private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile,
String errFile) throws Exception {
String[] cmdArray = args.toArray(new String[args.size()]);
return executeCmd(cmdArray, outFile, errFile);
}
private static QTestProcessExecResult executeCmd(String[] args, String outFile,
String errFile) throws Exception {
System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
PrintStream out = outFile == null ?
SessionState.getConsole().getChildOutStream() :
new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
PrintStream err = errFile == null ?
SessionState.getConsole().getChildErrStream() :
new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
Process executor = Runtime.getRuntime().exec(args);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream str = new PrintStream(bos, true, "UTF-8");
StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str);
outPrinter.start();
errPrinter.start();
int result = executor.waitFor();
outPrinter.join();
errPrinter.join();
if (outFile != null) {
out.close();
}
if (errFile != null) {
err.close();
}
return QTestProcessExecResult.
create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
}
private static String getQuotedString(String str){
return str;
}
public ASTNode parseQuery(String tname) throws Exception {
return pd.parse(qMap.get(tname));
}
public void resetParser() throws SemanticException {
drv.init();
pd = new ParseDriver();
queryState = new QueryState.Builder().withHiveConf(conf).build();
sem = new SemanticAnalyzer(queryState);
}
public List<Task<? extends Serializable>> analyzeAST(ASTNode ast) throws Exception {
// Do semantic analysis and plan generation
Context ctx = new Context(conf);
while ((ast.getToken() == null) && (ast.getChildCount() > 0)) {
ast = (ASTNode) ast.getChild(0);
}
sem.getOutputs().clear();
sem.getInputs().clear();
sem.analyze(ast, ctx);
ctx.clear();
return sem.getRootTasks();
}
public TreeMap<String, String> getQMap() {
return qMap;
}
/**
* QTestSetup defines test fixtures which are reused across testcases,
* and are needed before any test can be run
*/
public static class QTestSetup
{
private MiniZooKeeperCluster zooKeeperCluster = null;
private int zkPort;
private ZooKeeper zooKeeper;
public QTestSetup() {
}
public void preTest(HiveConf conf) throws Exception {
if (zooKeeperCluster == null) {
//create temp dir
String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
File tmpDir = Utilities.createTempDir(tmpBaseDir);
zooKeeperCluster = new MiniZooKeeperCluster();
zkPort = zooKeeperCluster.startup(tmpDir);
}
if (zooKeeper != null) {
zooKeeper.close();
}
int sessionTimeout = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, new Watcher() {
@Override
public void process(WatchedEvent arg0) {
}
});
String zkServer = "localhost";
conf.set("hive.zookeeper.quorum", zkServer);
conf.set("hive.zookeeper.client.port", "" + zkPort);
}
public void postTest(HiveConf conf) throws Exception {
if (zooKeeperCluster == null) {
return;
}
if (zooKeeper != null) {
zooKeeper.close();
}
ZooKeeperHiveLockManager.releaseAllLocks(conf);
}
public void tearDown() throws Exception {
CuratorFrameworkSingleton.closeAndReleaseInstance();
if (zooKeeperCluster != null) {
zooKeeperCluster.shutdown();
zooKeeperCluster = null;
}
}
}
/**
* QTRunner: Runnable class for running a a single query file.
*
**/
public static class QTRunner implements Runnable {
private final QTestUtil qt;
private final String fname;
public QTRunner(QTestUtil qt, String fname) {
this.qt = qt;
this.fname = fname;
}
@Override
public void run() {
try {
// assumption is that environment has already been cleaned once globally
// hence each thread does not call cleanUp() and createSources() again
qt.cliInit(fname, false);
qt.executeClient(fname);
} catch (Throwable e) {
System.err.println("Query file " + fname + " failed with exception "
+ e.getMessage());
e.printStackTrace();
outputTestFailureHelpMessage();
}
}
}
/**
* Setup to execute a set of query files. Uses QTestUtil to do so.
*
* @param qfiles
* array of input query files containing arbitrary number of hive
* queries
* @param resDir
* output directory
* @param logDir
* log directory
* @return one QTestUtil for each query file
*/
public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir,
String logDir, String initScript, String cleanupScript) throws Exception
{
QTestUtil[] qt = new QTestUtil[qfiles.length];
for (int i = 0; i < qfiles.length; i++) {
qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20",
initScript == null ? defaultInitScript : initScript,
cleanupScript == null ? defaultCleanupScript : cleanupScript,
false, false);
qt[i].addFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
return qt;
}
/**
* Executes a set of query files in sequence.
*
* @param qfiles
* array of input query files containing arbitrary number of hive
* queries
* @param qt
* array of QTestUtils, one per qfile
* @return true if all queries passed, false otw
*/
public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] qt)
throws Exception
{
boolean failed = false;
qt[0].cleanUp();
qt[0].createSources();
for (int i = 0; i < qfiles.length && !failed; i++) {
qt[i].clearTestSideEffects();
qt[i].cliInit(qfiles[i].getName(), false);
qt[i].executeClient(qfiles[i].getName());
QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
if (result.getReturnCode() != 0) {
failed = true;
StringBuilder builder = new StringBuilder();
builder.append("Test ")
.append(qfiles[i].getName())
.append(" results check failed with error code ")
.append(result.getReturnCode());
if (Strings.isNotEmpty(result.getCapturedOutput())) {
builder.append(" and diff value ").append(result.getCapturedOutput());
}
System.err.println(builder.toString());
outputTestFailureHelpMessage();
}
qt[i].clearPostTestEffects();
}
return (!failed);
}
/**
* Executes a set of query files parallel.
*
* Each query file is run in a separate thread. The caller has to arrange
* that different query files do not collide (in terms of destination tables)
*
* @param qfiles
* array of input query files containing arbitrary number of hive
* queries
* @param qt
* array of QTestUtils, one per qfile
* @return true if all queries passed, false otw
*
*/
public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt)
throws Exception
{
boolean failed = false;
// in multithreaded mode - do cleanup/initialization just once
qt[0].cleanUp();
qt[0].createSources();
qt[0].clearTestSideEffects();
QTRunner[] qtRunners = new QTRunner[qfiles.length];
Thread[] qtThread = new Thread[qfiles.length];
for (int i = 0; i < qfiles.length; i++) {
qtRunners[i] = new QTRunner(qt[i], qfiles[i].getName());
qtThread[i] = new Thread(qtRunners[i]);
}
for (int i = 0; i < qfiles.length; i++) {
qtThread[i].start();
}
for (int i = 0; i < qfiles.length; i++) {
qtThread[i].join();
QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
if (result.getReturnCode() != 0) {
failed = true;
StringBuilder builder = new StringBuilder();
builder.append("Test ")
.append(qfiles[i].getName())
.append(" results check failed with error code ")
.append(result.getReturnCode());
if (Strings.isNotEmpty(result.getCapturedOutput())) {
builder.append(" and diff value ").append(result.getCapturedOutput());
}
System.err.println(builder.toString());
outputTestFailureHelpMessage();
}
}
return (!failed);
}
public static void outputTestFailureHelpMessage() {
System.err.println(
"See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check " +
"./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific " +
"test cases logs.");
System.err.flush();
}
public static String ensurePathEndsInSlash(String path) {
if(path == null) {
throw new NullPointerException("Path cannot be null");
}
if(path.endsWith(File.separator)) {
return path;
} else {
return path + File.separator;
}
}
private static String[] cachedQvFileList = null;
private static ImmutableList<String> cachedDefaultQvFileList = null;
private static Pattern qvSuffix = Pattern.compile("_[0-9]+.qv$", Pattern.CASE_INSENSITIVE);
public static List<String> getVersionFiles(String queryDir, String tname) {
ensureQvFileList(queryDir);
List<String> result = getVersionFilesInternal(tname);
if (result == null) {
result = cachedDefaultQvFileList;
}
return result;
}
private static void ensureQvFileList(String queryDir) {
if (cachedQvFileList != null) return;
// Not thread-safe.
System.out.println("Getting versions from " + queryDir);
cachedQvFileList = (new File(queryDir)).list(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.toLowerCase().endsWith(".qv");
}
});
if (cachedQvFileList == null) return; // no files at all
Arrays.sort(cachedQvFileList, String.CASE_INSENSITIVE_ORDER);
List<String> defaults = getVersionFilesInternal("default");
cachedDefaultQvFileList = (defaults != null)
? ImmutableList.copyOf(defaults) : ImmutableList.<String>of();
}
private static List<String> getVersionFilesInternal(String tname) {
if (cachedQvFileList == null) {
return new ArrayList<String>();
}
int pos = Arrays.binarySearch(cachedQvFileList, tname, String.CASE_INSENSITIVE_ORDER);
if (pos >= 0) {
throw new BuildException("Unexpected file list element: " + cachedQvFileList[pos]);
}
List<String> result = null;
for (pos = (-pos - 1); pos < cachedQvFileList.length; ++pos) {
String candidate = cachedQvFileList[pos];
if (candidate.length() <= tname.length()
|| !tname.equalsIgnoreCase(candidate.substring(0, tname.length()))
|| !qvSuffix.matcher(candidate.substring(tname.length())).matches()) {
break;
}
if (result == null) {
result = new ArrayList<String>();
}
result.add(candidate);
}
return result;
}
public void failed(int ecode, String fname, String debugHint) {
String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
String message = "Client execution failed with error code = " + ecode +
(command != null ? " running \"" + command : "") + "\" fname=" + fname + " " +
(debugHint != null ? debugHint : "");
LOG.error(message);
Assert.fail(message);
}
// for negative tests, which is succeeded.. no need to print the query string
public void failed(String fname, String debugHint) {
Assert.fail(
"Client Execution was expected to fail, but succeeded with error code 0 for fname=" +
fname + (debugHint != null ? (" " + debugHint) : ""));
}
public void failedDiff(int ecode, String fname, String debugHint) {
String message =
"Client Execution succeeded but contained differences " +
"(error code = " + ecode + ") after executing " +
fname + (debugHint != null ? (" " + debugHint) : "");
LOG.error(message);
Assert.fail(message);
}
public void failed(Exception e, String fname, String debugHint) {
String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
System.err.println("Failed query: " + fname);
System.err.flush();
Assert.fail("Unexpected exception " +
org.apache.hadoop.util.StringUtils.stringifyException(e) + "\n" +
(command != null ? " running " + command : "") +
(debugHint != null ? debugHint : ""));
}
public static void addTestsToSuiteFromQfileNames(
String qFileNamesFile,
Set<String> qFilesToExecute,
TestSuite suite,
Object setup,
SuiteAddTestFunctor suiteAddTestCallback) {
try {
File qFileNames = new File(qFileNamesFile);
FileReader fr = new FileReader(qFileNames.getCanonicalFile());
BufferedReader br = new BufferedReader(fr);
String fName = null;
while ((fName = br.readLine()) != null) {
if (fName.isEmpty() || fName.trim().equals("")) {
continue;
}
int eIdx = fName.indexOf('.');
if (eIdx == -1) {
continue;
}
String tName = fName.substring(0, eIdx);
if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) {
suiteAddTestCallback.addTestToSuite(suite, setup, tName);
}
}
br.close();
} catch (Exception e) {
Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
}
}
public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf conf) {
Connection conn = null;
ArrayList<Statement> statements = new ArrayList<Statement>(); // list of Statements, PreparedStatements
try {
Properties props = new Properties(); // connection properties
props.put("user", conf.get("javax.jdo.option.ConnectionUserName"));
props.put("password", conf.get("javax.jdo.option.ConnectionPassword"));
conn = DriverManager.getConnection(conf.get("javax.jdo.option.ConnectionURL"), props);
ResultSet rs = null;
Statement s = conn.createStatement();
if (LOG.isDebugEnabled()) {
LOG.debug("Connected to metastore database ");
}
String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
// Setup the table column stats
BufferedReader br = new BufferedReader(
new FileReader(
new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
String command;
s.execute("DROP TABLE APP.TABLE_PARAMS");
s.execute("DROP TABLE APP.TAB_COL_STATS");
// Create the column stats table
while ((command = br.readLine()) != null) {
if (!command.endsWith(";")) {
continue;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Going to run command : " + command);
}
try {
PreparedStatement psCommand = conn.prepareStatement(command.substring(0, command.length()-1));
statements.add(psCommand);
psCommand.execute();
if (LOG.isDebugEnabled()) {
LOG.debug("successfully completed " + command);
}
} catch (SQLException e) {
LOG.info("Got SQL Exception " + e.getMessage());
}
}
br.close();
java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2");
java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2");
// Set up the foreign key constraints properly in the TAB_COL_STATS data
String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt");
java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt");
class MyComp implements Comparator<String> {
@Override
public int compare(String str1, String str2) {
if (str2.length() != str1.length()) {
return str2.length() - str1.length();
}
return str1.compareTo(str2);
}
}
final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
rs = s.executeQuery("SELECT * FROM APP.TBLS");
while(rs.next()) {
String tblName = rs.getString("TBL_NAME");
Integer tblId = rs.getInt("TBL_ID");
tableNameToID.put(tblName, tblId);
if (LOG.isDebugEnabled()) {
LOG.debug("Resultset : " + tblName + " | " + tblId);
}
}
final Map<String, Map<String, String>> data = new HashMap<>();
rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from "
+ "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a"
+ " join APP.TBLS on TBLS.SD_ID = a.SD_ID");
while (rs.next()) {
String tblName = rs.getString(1);
String colName = rs.getString(2);
String typeName = rs.getString(3);
Map<String, String> cols = data.get(tblName);
if (null == cols) {
cols = new HashMap<>();
}
cols.put(colName, typeName);
data.put(tblName, cols);
}
BufferedReader reader = new BufferedReader(new InputStreamReader(
new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ))));
Stream<String> replaced = reader.lines().parallel().map(str-> {
String[] splits = str.split(",");
String tblName = splits[0];
String colName = splits[1];
Integer tblID = tableNameToID.get(tblName);
StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@");
for (int i = 2; i < splits.length; i++) {
sb.append(splits[i]+"@");
}
return sb.append(tblID).toString();
});
Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator);
replaced.close();
reader.close();
BufferedReader reader2 = new BufferedReader(new InputStreamReader(
new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ))));
final Map<String,String> colStats = new ConcurrentHashMap<>();
Stream<String> replacedStream = reader2.lines().parallel().map(str-> {
String[] splits = str.split("_@");
String tblName = splits[0];
Integer tblId = tableNameToID.get(tblName);
Map<String,String> cols = data.get(tblName);
StringBuilder sb = new StringBuilder();
sb.append("{\"COLUMN_STATS\":{");
for (String colName : cols.keySet()) {
sb.append("\""+colName+"\":\"true\",");
}
sb.append("},\"BASIC_STATS\":\"true\"}");
colStats.put(tblId.toString(), sb.toString());
return tblId.toString() + "@" + splits[1];
});
Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator);
Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream()
.map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND);
replacedStream.close();
reader2.close();
// Load the column stats and table params with 30 TB scale
String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" +
"', '" + tmpFileLoc1.toAbsolutePath().toString() +
"', '@', null, 'UTF-8', 1)";
String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" +
"', '" + tmpFileLoc2.toAbsolutePath().toString() +
"', '@', null, 'UTF-8', 1)";
try {
PreparedStatement psImport1 = conn.prepareStatement(importStatement1);
if (LOG.isDebugEnabled()) {
LOG.debug("Going to execute : " + importStatement1);
}
statements.add(psImport1);
psImport1.execute();
if (LOG.isDebugEnabled()) {
LOG.debug("successfully completed " + importStatement1);
}
PreparedStatement psImport2 = conn.prepareStatement(importStatement2);
if (LOG.isDebugEnabled()) {
LOG.debug("Going to execute : " + importStatement2);
}
statements.add(psImport2);
psImport2.execute();
if (LOG.isDebugEnabled()) {
LOG.debug("successfully completed " + importStatement2);
}
} catch (SQLException e) {
LOG.info("Got SQL Exception " + e.getMessage());
}
} catch (FileNotFoundException e1) {
LOG.info("Got File not found Exception " + e1.getMessage());
} catch (IOException e1) {
LOG.info("Got IOException " + e1.getMessage());
} catch (SQLException e1) {
LOG.info("Got SQLException " + e1.getMessage());
} finally {
// Statements and PreparedStatements
int i = 0;
while (!statements.isEmpty()) {
// PreparedStatement extend Statement
Statement st = statements.remove(i);
try {
if (st != null) {
st.close();
st = null;
}
} catch (SQLException sqle) {
}
}
//Connection
try {
if (conn != null) {
conn.close();
conn = null;
}
} catch (SQLException sqle) {
}
}
}
}