/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.hive;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
/**
* Hive interpreter for Zeppelin.
*/
public class HiveInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(HiveInterpreter.class);
static final String COMMON_KEY = "common";
static final String MAX_LINE_KEY = "max_count";
static final String MAX_LINE_DEFAULT = "1000";
static final String DEFAULT_KEY = "default";
static final String DRIVER_KEY = "driver";
static final String URL_KEY = "url";
static final String USER_KEY = "user";
static final String PASSWORD_KEY = "password";
static final String DOT = ".";
static final char TAB = '\t';
static final char NEWLINE = '\n';
static final String EXPLAIN_PREDICATE = "EXPLAIN ";
static final String TABLE_MAGIC_TAG = "%table ";
static final String UPDATE_COUNT_HEADER = "Update Count";
static final String COMMON_MAX_LINE = COMMON_KEY + DOT + MAX_LINE_KEY;
static final String DEFAULT_DRIVER = DEFAULT_KEY + DOT + DRIVER_KEY;
static final String DEFAULT_URL = DEFAULT_KEY + DOT + URL_KEY;
static final String DEFAULT_USER = DEFAULT_KEY + DOT + USER_KEY;
static final String DEFAULT_PASSWORD = DEFAULT_KEY + DOT + PASSWORD_KEY;
private final HashMap<String, Properties> propertiesMap;
private final Map<String, Statement> paragraphIdStatementMap;
private final Map<String, ArrayList<Connection>> propertyKeyUnusedConnectionListMap;
private final Map<String, Connection> paragraphIdConnectionMap;
static {
Interpreter.register(
"hql",
"hive",
HiveInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(COMMON_MAX_LINE, MAX_LINE_DEFAULT, "Maximum line of results")
.add(DEFAULT_DRIVER, "org.apache.hive.jdbc.HiveDriver", "Hive JDBC driver")
.add(DEFAULT_URL, "jdbc:hive2://localhost:10000", "The URL for HiveServer2.")
.add(DEFAULT_USER, "hive", "The hive user")
.add(DEFAULT_PASSWORD, "", "The password for the hive user").build());
}
public HiveInterpreter(Properties property) {
super(property);
propertiesMap = new HashMap<>();
propertyKeyUnusedConnectionListMap = new HashMap<>();
paragraphIdStatementMap = new HashMap<>();
paragraphIdConnectionMap = new HashMap<>();
}
public HashMap<String, Properties> getPropertiesMap() {
return propertiesMap;
}
@Override
public void open() {
logger.debug("property: {}", property);
for (String propertyKey : property.stringPropertyNames()) {
logger.debug("propertyKey: {}", propertyKey);
String[] keyValue = propertyKey.split("\\.", 2);
if (2 == keyValue.length) {
logger.debug("key: {}, value: {}", keyValue[0], keyValue[1]);
Properties prefixProperties;
if (propertiesMap.containsKey(keyValue[0])) {
prefixProperties = propertiesMap.get(keyValue[0]);
} else {
prefixProperties = new Properties();
propertiesMap.put(keyValue[0], prefixProperties);
}
prefixProperties.put(keyValue[1], property.getProperty(propertyKey));
}
}
Set<String> removeKeySet = new HashSet<>();
for (String key : propertiesMap.keySet()) {
if (!COMMON_KEY.equals(key)) {
Properties properties = propertiesMap.get(key);
if (!properties.containsKey(DRIVER_KEY) || !properties.containsKey(URL_KEY)) {
logger.error("{} will be ignored. {}.{} and {}.{} is mandatory.",
key, DRIVER_KEY, key, key, URL_KEY);
removeKeySet.add(key);
}
}
}
for (String key : removeKeySet) {
propertiesMap.remove(key);
}
logger.debug("propertiesMap: {}", propertiesMap);
}
@Override
public void close() {
try {
for (List<Connection> connectionList : propertyKeyUnusedConnectionListMap.values()) {
for (Connection c : connectionList) {
c.close();
}
}
for (Statement statement : paragraphIdStatementMap.values()) {
statement.close();
}
paragraphIdStatementMap.clear();
for (Connection connection : paragraphIdConnectionMap.values()) {
connection.close();
}
paragraphIdConnectionMap.clear();
} catch (SQLException e) {
logger.error("Error while closing...", e);
}
}
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
Connection connection = null;
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
ArrayList<Connection> connectionList = propertyKeyUnusedConnectionListMap.get(propertyKey);
if (0 != connectionList.size()) {
connection = propertyKeyUnusedConnectionListMap.get(propertyKey).remove(0);
if (null != connection && connection.isClosed()) {
connection.close();
connection = null;
}
}
}
if (null == connection) {
Properties properties = propertiesMap.get(propertyKey);
Class.forName(properties.getProperty(DRIVER_KEY));
String url = properties.getProperty(URL_KEY);
String user = properties.getProperty(USER_KEY);
String password = properties.getProperty(PASSWORD_KEY);
if (null != user && null != password) {
connection = DriverManager.getConnection(url, user, password);
} else {
connection = DriverManager.getConnection(url, properties);
}
}
return connection;
}
public Statement getStatement(String propertyKey, String paragraphId)
throws SQLException, ClassNotFoundException {
Connection connection;
if (paragraphIdConnectionMap.containsKey(paragraphId)) {
// Never enter for now.
connection = paragraphIdConnectionMap.get(paragraphId);
} else {
connection = getConnection(propertyKey);
}
Statement statement = connection.createStatement();
if (isStatementClosed(statement)) {
connection = getConnection(propertyKey);
statement = connection.createStatement();
}
paragraphIdConnectionMap.put(paragraphId, connection);
paragraphIdStatementMap.put(paragraphId, statement);
return statement;
}
private boolean isStatementClosed(Statement statement) {
try {
return statement.isClosed();
} catch (Throwable t) {
logger.debug("{} doesn't support isClosed method", statement);
return false;
}
}
public InterpreterResult executeSql(String propertyKey, String sql,
InterpreterContext interpreterContext) {
String paragraphId = interpreterContext.getParagraphId();
try {
Statement statement = getStatement(propertyKey, paragraphId);
statement.setMaxRows(getMaxResult());
StringBuilder msg;
if (containsIgnoreCase(sql, EXPLAIN_PREDICATE)) {
msg = new StringBuilder();
} else {
msg = new StringBuilder(TABLE_MAGIC_TAG);
}
ResultSet resultSet = null;
try {
boolean isResultSetAvailable = statement.execute(sql);
if (isResultSetAvailable) {
resultSet = statement.getResultSet();
ResultSetMetaData md = resultSet.getMetaData();
for (int i = 1; i < md.getColumnCount() + 1; i++) {
if (i > 1) {
msg.append(TAB);
}
msg.append(md.getColumnName(i));
}
msg.append(NEWLINE);
int displayRowCount = 0;
while (resultSet.next() && displayRowCount < getMaxResult()) {
for (int i = 1; i < md.getColumnCount() + 1; i++) {
msg.append(resultSet.getString(i));
if (i != md.getColumnCount()) {
msg.append(TAB);
}
}
msg.append(NEWLINE);
displayRowCount++;
}
} else {
// Response contains either an update count or there are no results.
int updateCount = statement.getUpdateCount();
msg.append(UPDATE_COUNT_HEADER).append(NEWLINE);
msg.append(updateCount).append(NEWLINE);
}
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
statement.close();
} finally {
moveConnectionToUnused(propertyKey, paragraphId);
}
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
} catch (SQLException | ClassNotFoundException ex) {
logger.error("Cannot run " + sql, ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
}
private void moveConnectionToUnused(String propertyKey, String paragraphId) {
if (paragraphIdConnectionMap.containsKey(paragraphId)) {
Connection connection = paragraphIdConnectionMap.remove(paragraphId);
if (null != connection) {
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
propertyKeyUnusedConnectionListMap.get(propertyKey).add(connection);
} else {
ArrayList<Connection> connectionList = new ArrayList<>();
connectionList.add(connection);
propertyKeyUnusedConnectionListMap.put(propertyKey, connectionList);
}
}
}
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
String propertyKey = getPropertyKey(cmd);
if (null != propertyKey) {
cmd = cmd.substring(propertyKey.length() + 2);
} else {
propertyKey = DEFAULT_KEY;
}
cmd = cmd.trim();
logger.info("PropertyKey: {}, SQL command: '{}'", propertyKey, cmd);
return executeSql(propertyKey, cmd, contextInterpreter);
}
private int getMaxResult() {
return Integer.valueOf(
propertiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
}
public String getPropertyKey(String cmd) {
int firstLineIndex = cmd.indexOf("\n");
if (-1 == firstLineIndex) {
firstLineIndex = cmd.length();
}
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1
&& configLastIndex < firstLineIndex && configLastIndex < firstLineIndex) {
return cmd.substring(configStartIndex + 1, configLastIndex);
}
return null;
}
@Override
public void cancel(InterpreterContext context) {
String paragraphId = context.getParagraphId();
try {
paragraphIdStatementMap.get(paragraphId).cancel();
} catch (SQLException e) {
logger.error("Error while cancelling...", e);
}
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetParallelScheduler(
HiveInterpreter.class.getName() + this.hashCode(), 10);
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}