/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.github.sakserv.minicluster.impl;
import java.io.File;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.sakserv.minicluster.MiniCluster;
import com.github.sakserv.minicluster.util.FileUtils;
import com.github.sakserv.minicluster.util.WindowsLibsUtils;
public class HiveLocalMetaStore implements MiniCluster {
// Logger
private static final Logger LOG = LoggerFactory.getLogger(HiveLocalMetaStore.class);
private String hiveMetastoreHostname;
private Integer hiveMetastorePort;
private String hiveMetastoreDerbyDbDir;
private String hiveScratchDir;
private String hiveWarehouseDir;
private HiveConf hiveConf;
private Thread t;
private HiveLocalMetaStore(Builder builder) {
this.hiveMetastoreHostname = builder.hiveMetastoreHostname;
this.hiveMetastorePort = builder.hiveMetastorePort;
this.hiveMetastoreDerbyDbDir = builder.hiveMetastoreDerbyDbDir;
this.hiveScratchDir = builder.hiveScratchDir;
this.hiveWarehouseDir = builder.hiveWarehouseDir;
this.hiveConf = builder.hiveConf;
}
public String getHiveMetastoreHostname() {
return hiveMetastoreHostname;
}
public Integer getHiveMetastorePort() {
return hiveMetastorePort;
}
public String getHiveMetastoreDerbyDbDir() {
return hiveMetastoreDerbyDbDir;
}
public String getHiveScratchDir() {
return hiveScratchDir;
}
public HiveConf getHiveConf() {
return hiveConf;
}
public String getHiveWarehouseDir() {
return hiveWarehouseDir;
}
public static class Builder {
private String hiveMetastoreHostname;
private Integer hiveMetastorePort;
private String hiveMetastoreDerbyDbDir;
private String hiveScratchDir;
private String hiveWarehouseDir;
private HiveConf hiveConf;
public Builder setHiveMetastoreHostname(String hiveMetastoreHostname) {
this.hiveMetastoreHostname = hiveMetastoreHostname;
return this;
}
public Builder setHiveMetastorePort(int hiveMetaStorePort) {
this.hiveMetastorePort = hiveMetaStorePort;
return this;
}
public Builder setHiveMetastoreDerbyDbDir(String hiveDerbyDbDir) {
this.hiveMetastoreDerbyDbDir = hiveDerbyDbDir;
return this;
}
public Builder setHiveScratchDir(String hiveScratchDir) {
this.hiveScratchDir = hiveScratchDir;
return this;
}
public Builder setHiveConf(HiveConf hiveConf) {
this.hiveConf = hiveConf;
return this;
}
public Builder setHiveWarehouseDir(String hiveWarehouseDir) {
this.hiveWarehouseDir = hiveWarehouseDir;
return this;
}
public HiveLocalMetaStore build() {
HiveLocalMetaStore hiveLocalMetaStore = new HiveLocalMetaStore(this);
validateObject(hiveLocalMetaStore);
return hiveLocalMetaStore;
}
public void validateObject(HiveLocalMetaStore hiveLocalMetaStore) {
if(hiveLocalMetaStore.hiveMetastoreHostname == null) {
throw new IllegalArgumentException("ERROR: Missing required config: Hive Meta Store Hostname");
}
if(hiveLocalMetaStore.hiveMetastorePort == null) {
throw new IllegalArgumentException("ERROR: Missing required config: Hive Meta Store Port");
}
if(hiveLocalMetaStore.hiveMetastoreDerbyDbDir == null) {
throw new IllegalArgumentException("ERROR: Missing required config: Hive Derby Db Path");
}
if(hiveLocalMetaStore.hiveScratchDir == null) {
throw new IllegalArgumentException("ERROR: Missing required config: Hive Scratch Dir");
}
if(hiveLocalMetaStore.hiveWarehouseDir == null) {
throw new IllegalArgumentException("ERROR: Missing required config: Hive Warehouse Dir");
}
if(hiveLocalMetaStore.hiveConf == null) {
throw new IllegalArgumentException("ERROR: Missing required config: Hive Conf");
}
}
}
private static class StartHiveLocalMetaStore implements Runnable {
private Integer hiveMetastorePort;
private HiveConf hiveConf;
public void setHiveMetastorePort(Integer hiveMetastorePort) {
this.hiveMetastorePort = hiveMetastorePort;
}
public void setHiveConf(HiveConf hiveConf) {
this.hiveConf = hiveConf;
}
@Override
public void run() {
try {
HiveMetaStore.startMetaStore(hiveMetastorePort,
new HadoopThriftAuthBridge(),
hiveConf);
} catch (Throwable t) {
t.printStackTrace();
}
}
}
@Override
public void start() throws Exception {
LOG.info("HIVEMETASTORE: Starting Hive Metastore on port: {}", hiveMetastorePort);
configure();
StartHiveLocalMetaStore startHiveLocalMetaStore = new StartHiveLocalMetaStore();
startHiveLocalMetaStore.setHiveMetastorePort(hiveMetastorePort);
startHiveLocalMetaStore.setHiveConf(hiveConf);
t = new Thread(startHiveLocalMetaStore);
t.setDaemon(true);
t.start();
}
@Override
public void stop() throws Exception {
stop(true);
}
@Override
public void stop(boolean cleanUp) throws Exception {
LOG.info("HIVEMETASTORE: Stopping Hive Metastore on port: {}", hiveMetastorePort);
t.interrupt();
if (cleanUp) {
cleanUp();
}
}
@Override
public void configure() throws Exception {
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS,
"thrift://" + hiveMetastoreHostname + ":" + hiveMetastorePort);
hiveConf.setVar(HiveConf.ConfVars.SCRATCHDIR, hiveScratchDir);
hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
"jdbc:derby:;databaseName=" + hiveMetastoreDerbyDbDir + ";create=true");
hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, new File(hiveWarehouseDir).getAbsolutePath());
hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true);
hiveConf.set("datanucleus.schema.autoCreateTables", "true");
hiveConf.set("hive.metastore.schema.verification", "false");
// Handle Windows
WindowsLibsUtils.setHadoopHome();
}
@Override
public void cleanUp() throws Exception {
FileUtils.deleteFolder(hiveMetastoreDerbyDbDir);
FileUtils.deleteFolder(hiveWarehouseDir);
FileUtils.deleteFolder(new File("derby.log").getAbsolutePath());
}
}