/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hive.jdbc;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
import org.apache.hive.service.auth.HiveAuthConstants;
import org.apache.hive.service.auth.KerberosSaslHelper;
import org.apache.hive.service.auth.PlainSaslHelper;
import org.apache.hive.service.auth.SaslQOP;
import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService;
import org.apache.hive.service.rpc.thrift.TCLIService;
import org.apache.hive.service.rpc.thrift.TCancelDelegationTokenReq;
import org.apache.hive.service.rpc.thrift.TCancelDelegationTokenResp;
import org.apache.hive.service.rpc.thrift.TCloseSessionReq;
import org.apache.hive.service.rpc.thrift.TGetDelegationTokenReq;
import org.apache.hive.service.rpc.thrift.TGetDelegationTokenResp;
import org.apache.hive.service.rpc.thrift.TOpenSessionReq;
import org.apache.hive.service.rpc.thrift.TOpenSessionResp;
import org.apache.hive.service.rpc.thrift.TProtocolVersion;
import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenReq;
import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenResp;
import org.apache.hive.service.rpc.thrift.TSessionHandle;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.ServiceUnavailableRetryStrategy;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContexts;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.THttpClient;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslException;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.sql.Array;
import java.sql.Blob;
import java.sql.CallableStatement;
import java.sql.Clob;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.NClob;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLClientInfoException;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.sql.SQLWarning;
import java.sql.SQLXML;
import java.sql.Savepoint;
import java.sql.Statement;
import java.sql.Struct;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
* HiveConnection.
*
*/
public class HiveConnection implements java.sql.Connection {
public static final Logger LOG = LoggerFactory.getLogger(HiveConnection.class.getName());
private String jdbcUriString;
private String host;
private int port;
private final Map<String, String> sessConfMap;
private JdbcConnectionParams connParams;
private final boolean isEmbeddedMode;
private TTransport transport;
private boolean assumeSubject;
// TODO should be replaced by CliServiceClient
private TCLIService.Iface client;
private boolean isClosed = true;
private SQLWarning warningChain = null;
private TSessionHandle sessHandle = null;
private final List<TProtocolVersion> supportedProtocols = new LinkedList<TProtocolVersion>();
private int loginTimeout = 0;
private TProtocolVersion protocol;
private int fetchSize = HiveStatement.DEFAULT_FETCH_SIZE;
private String initFile = null;
public HiveConnection(String uri, Properties info) throws SQLException {
setupLoginTimeout();
try {
connParams = Utils.parseURL(uri, info);
} catch (ZooKeeperHiveClientException e) {
throw new SQLException(e);
}
jdbcUriString = connParams.getJdbcUriString();
// JDBC URL: jdbc:hive2://<host>:<port>/dbName;sess_var_list?hive_conf_list#hive_var_list
// each list: <key1>=<val1>;<key2>=<val2> and so on
// sess_var_list -> sessConfMap
// hive_conf_list -> hiveConfMap
// hive_var_list -> hiveVarMap
host = connParams.getHost();
port = connParams.getPort();
sessConfMap = connParams.getSessionVars();
isEmbeddedMode = connParams.isEmbeddedMode();
if (sessConfMap.containsKey(JdbcConnectionParams.FETCH_SIZE)) {
fetchSize = Integer.parseInt(sessConfMap.get(JdbcConnectionParams.FETCH_SIZE));
}
if (sessConfMap.containsKey(JdbcConnectionParams.INIT_FILE)) {
initFile = sessConfMap.get(JdbcConnectionParams.INIT_FILE);
}
// add supported protocols
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V2);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V3);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V4);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V5);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V7);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V9);
supportedProtocols.add(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10);
if (isEmbeddedMode) {
EmbeddedThriftBinaryCLIService embeddedClient = new EmbeddedThriftBinaryCLIService();
embeddedClient.init(null);
client = embeddedClient;
// open client session
openSession();
executeInitSql();
} else {
int maxRetries = 1;
try {
String strRetries = sessConfMap.get(JdbcConnectionParams.RETRIES);
if (StringUtils.isNotBlank(strRetries)) {
maxRetries = Integer.parseInt(strRetries);
}
} catch(NumberFormatException e) { // Ignore the exception
}
for (int numRetries = 0;;) {
try {
// open the client transport
openTransport();
// set up the client
client = new TCLIService.Client(new TBinaryProtocol(transport));
// open client session
openSession();
executeInitSql();
break;
} catch (Exception e) {
LOG.warn("Failed to connect to " + connParams.getHost() + ":" + connParams.getPort());
String errMsg = null;
String warnMsg = "Could not open client transport with JDBC Uri: " + jdbcUriString + ": ";
if (isZkDynamicDiscoveryMode()) {
errMsg = "Could not open client transport for any of the Server URI's in ZooKeeper: ";
// Try next available server in zookeeper, or retry all the servers again if retry is enabled
while(!Utils.updateConnParamsFromZooKeeper(connParams) && ++numRetries < maxRetries) {
connParams.getRejectedHostZnodePaths().clear();
}
// Update with new values
jdbcUriString = connParams.getJdbcUriString();
host = connParams.getHost();
port = connParams.getPort();
} else {
errMsg = warnMsg;
++numRetries;
}
if (numRetries >= maxRetries) {
throw new SQLException(errMsg + e.getMessage(), " 08S01", e);
} else {
LOG.warn(warnMsg + e.getMessage() + " Retrying " + numRetries + " of " + maxRetries);
}
}
}
}
// Wrap the client with a thread-safe proxy to serialize the RPC calls
client = newSynchronizedClient(client);
}
private void executeInitSql() throws SQLException {
if (initFile != null) {
try {
List<String> sqlList = parseInitFile(initFile);
Statement st = createStatement();
for(String sql : sqlList) {
boolean hasResult = st.execute(sql);
if (hasResult) {
ResultSet rs = st.getResultSet();
while (rs.next()) {
System.out.println(rs.getString(1));
}
}
}
} catch(Exception e) {
LOG.error("Failed to execute initial SQL");
throw new SQLException(e.getMessage());
}
}
}
public static List<String> parseInitFile(String initFile) throws IOException {
File file = new File(initFile);
BufferedReader br = null;
List<String> initSqlList = null;
try {
FileInputStream input = new FileInputStream(file);
br = new BufferedReader(new InputStreamReader(input, "UTF-8"));
String line;
StringBuilder sb = new StringBuilder("");
while ((line = br.readLine()) != null) {
line = line.trim();
if (line.length() != 0) {
if (line.startsWith("#") || line.startsWith("--")) {
continue;
} else {
line = line.concat(" ");
sb.append(line);
}
}
}
initSqlList = getInitSql(sb.toString());
} catch(IOException e) {
LOG.error("Failed to read initial SQL file", e);
throw new IOException(e);
} finally {
if (br != null) {
br.close();
}
}
return initSqlList;
}
private static List<String> getInitSql(String sbLine) {
char[] sqlArray = sbLine.toCharArray();
List<String> initSqlList = new ArrayList();
int index = 0;
int beginIndex = 0;
for (; index < sqlArray.length; index++) {
if (sqlArray[index] == ';') {
String sql = sbLine.substring(beginIndex, index).trim();
initSqlList.add(sql);
beginIndex = index + 1;
}
}
return initSqlList;
}
private void openTransport() throws Exception {
assumeSubject =
JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT.equals(sessConfMap
.get(JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE));
transport = isHttpTransportMode() ? createHttpTransport() : createBinaryTransport();
if (!transport.isOpen()) {
transport.open();
}
logZkDiscoveryMessage("Connected to " + connParams.getHost() + ":" + connParams.getPort());
}
public String getConnectedUrl() {
return jdbcUriString;
}
private String getServerHttpUrl(boolean useSsl) {
// Create the http/https url
// JDBC driver will set up an https url if ssl is enabled, otherwise http
String schemeName = useSsl ? "https" : "http";
// http path should begin with "/"
String httpPath;
httpPath = sessConfMap.get(JdbcConnectionParams.HTTP_PATH);
if (httpPath == null) {
httpPath = "/";
} else if (!httpPath.startsWith("/")) {
httpPath = "/" + httpPath;
}
return schemeName + "://" + host + ":" + port + httpPath;
}
private TTransport createHttpTransport() throws SQLException, TTransportException {
CloseableHttpClient httpClient;
boolean useSsl = isSslConnection();
// Create an http client from the configs
httpClient = getHttpClient(useSsl);
transport = new THttpClient(getServerHttpUrl(useSsl), httpClient);
return transport;
}
private CloseableHttpClient getHttpClient(Boolean useSsl) throws SQLException {
boolean isCookieEnabled = sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH) == null ||
(!JdbcConnectionParams.COOKIE_AUTH_FALSE.equalsIgnoreCase(
sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH)));
String cookieName = sessConfMap.get(JdbcConnectionParams.COOKIE_NAME) == null ?
JdbcConnectionParams.DEFAULT_COOKIE_NAMES_HS2 :
sessConfMap.get(JdbcConnectionParams.COOKIE_NAME);
CookieStore cookieStore = isCookieEnabled ? new BasicCookieStore() : null;
HttpClientBuilder httpClientBuilder;
// Request interceptor for any request pre-processing logic
HttpRequestInterceptor requestInterceptor;
Map<String, String> additionalHttpHeaders = new HashMap<String, String>();
// Retrieve the additional HttpHeaders
for (Map.Entry<String, String> entry : sessConfMap.entrySet()) {
String key = entry.getKey();
if (key.startsWith(JdbcConnectionParams.HTTP_HEADER_PREFIX)) {
additionalHttpHeaders.put(key.substring(JdbcConnectionParams.HTTP_HEADER_PREFIX.length()),
entry.getValue());
}
}
// Configure http client for kerberos/password based authentication
if (isKerberosAuthMode()) {
/**
* Add an interceptor which sets the appropriate header in the request.
* It does the kerberos authentication and get the final service ticket,
* for sending to the server before every request.
* In https mode, the entire information is encrypted
*/
requestInterceptor =
new HttpKerberosRequestInterceptor(sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL),
host, getServerHttpUrl(useSsl), assumeSubject, cookieStore, cookieName, useSsl,
additionalHttpHeaders);
} else {
// Check for delegation token, if present add it in the header
String tokenStr = getClientDelegationToken(sessConfMap);
if (tokenStr != null) {
requestInterceptor =
new HttpTokenAuthInterceptor(tokenStr, cookieStore, cookieName, useSsl,
additionalHttpHeaders);
} else {
/**
* Add an interceptor to pass username/password in the header.
* In https mode, the entire information is encrypted
*/
requestInterceptor =
new HttpBasicAuthInterceptor(getUserName(), getPassword(), cookieStore, cookieName,
useSsl, additionalHttpHeaders);
}
}
// Configure http client for cookie based authentication
if (isCookieEnabled) {
// Create a http client with a retry mechanism when the server returns a status code of 401.
httpClientBuilder =
HttpClients.custom().setServiceUnavailableRetryStrategy(
new ServiceUnavailableRetryStrategy() {
@Override
public boolean retryRequest(final HttpResponse response, final int executionCount,
final HttpContext context) {
int statusCode = response.getStatusLine().getStatusCode();
boolean ret = statusCode == 401 && executionCount <= 1;
// Set the context attribute to true which will be interpreted by the request
// interceptor
if (ret) {
context.setAttribute(Utils.HIVE_SERVER2_RETRY_KEY,
Utils.HIVE_SERVER2_RETRY_TRUE);
}
return ret;
}
@Override
public long getRetryInterval() {
// Immediate retry
return 0;
}
});
} else {
httpClientBuilder = HttpClientBuilder.create();
}
// In case the server's idletimeout is set to a lower value, it might close it's side of
// connection. However we retry one more time on NoHttpResponseException
httpClientBuilder.setRetryHandler(new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount > 1) {
LOG.info("Retry attempts to connect to server exceeded.");
return false;
}
if (exception instanceof org.apache.http.NoHttpResponseException) {
LOG.info("Could not connect to the server. Retrying one more time.");
return true;
}
return false;
}
});
// Add the request interceptor to the client builder
httpClientBuilder.addInterceptorFirst(requestInterceptor);
// Add an interceptor to add in an XSRF header
httpClientBuilder.addInterceptorLast(new XsrfHttpRequestInterceptor());
// Configure http client for SSL
if (useSsl) {
String useTwoWaySSL = sessConfMap.get(JdbcConnectionParams.USE_TWO_WAY_SSL);
String sslTrustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE);
String sslTrustStorePassword = sessConfMap.get(
JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
KeyStore sslTrustStore;
SSLConnectionSocketFactory socketFactory;
SSLContext sslContext;
/**
* The code within the try block throws: SSLInitializationException, KeyStoreException,
* IOException, NoSuchAlgorithmException, CertificateException, KeyManagementException &
* UnrecoverableKeyException. We don't want the client to retry on any of these,
* hence we catch all and throw a SQLException.
*/
try {
if (useTwoWaySSL != null && useTwoWaySSL.equalsIgnoreCase(JdbcConnectionParams.TRUE)) {
socketFactory = getTwoWaySSLSocketFactory();
} else if (sslTrustStorePath == null || sslTrustStorePath.isEmpty()) {
// Create a default socket factory based on standard JSSE trust material
socketFactory = SSLConnectionSocketFactory.getSocketFactory();
} else {
// Pick trust store config from the given path
sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
try (FileInputStream fis = new FileInputStream(sslTrustStorePath)) {
sslTrustStore.load(fis, sslTrustStorePassword.toCharArray());
}
sslContext = SSLContexts.custom().loadTrustMaterial(sslTrustStore, null).build();
socketFactory =
new SSLConnectionSocketFactory(sslContext, new DefaultHostnameVerifier(null));
}
final Registry<ConnectionSocketFactory> registry =
RegistryBuilder.<ConnectionSocketFactory> create().register("https", socketFactory)
.build();
httpClientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry));
} catch (Exception e) {
String msg =
"Could not create an https connection to " + jdbcUriString + ". " + e.getMessage();
throw new SQLException(msg, " 08S01", e);
}
}
return httpClientBuilder.build();
}
/**
* Create underlying SSL or non-SSL transport
*
* @return TTransport
* @throws TTransportException
*/
private TTransport createUnderlyingTransport() throws TTransportException {
TTransport transport = null;
// Note: Thrift returns an SSL socket that is already bound to the specified host:port
// Therefore an open called on this would be a no-op later
// Hence, any TTransportException related to connecting with the peer are thrown here.
// Bubbling them up the call hierarchy so that a retry can happen in openTransport,
// if dynamic service discovery is configured.
if (isSslConnection()) {
// get SSL socket
String sslTrustStore = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE);
String sslTrustStorePassword = sessConfMap.get(
JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
if (sslTrustStore == null || sslTrustStore.isEmpty()) {
transport = HiveAuthUtils.getSSLSocket(host, port, loginTimeout);
} else {
transport = HiveAuthUtils.getSSLSocket(host, port, loginTimeout,
sslTrustStore, sslTrustStorePassword);
}
} else {
// get non-SSL socket transport
transport = HiveAuthUtils.getSocketTransport(host, port, loginTimeout);
}
return transport;
}
/**
* Create transport per the connection options
* Supported transport options are:
* - SASL based transports over
* + Kerberos
* + Delegation token
* + SSL
* + non-SSL
* - Raw (non-SASL) socket
*
* Kerberos and Delegation token supports SASL QOP configurations
* @throws SQLException, TTransportException
*/
private TTransport createBinaryTransport() throws SQLException, TTransportException {
try {
TTransport socketTransport = createUnderlyingTransport();
// handle secure connection if specified
if (!JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))) {
// If Kerberos
Map<String, String> saslProps = new HashMap<String, String>();
SaslQOP saslQOP = SaslQOP.AUTH;
if (sessConfMap.containsKey(JdbcConnectionParams.AUTH_QOP)) {
try {
saslQOP = SaslQOP.fromString(sessConfMap.get(JdbcConnectionParams.AUTH_QOP));
} catch (IllegalArgumentException e) {
throw new SQLException("Invalid " + JdbcConnectionParams.AUTH_QOP +
" parameter. " + e.getMessage(), "42000", e);
}
saslProps.put(Sasl.QOP, saslQOP.toString());
} else {
// If the client did not specify qop then just negotiate the one supported by server
saslProps.put(Sasl.QOP, "auth-conf,auth-int,auth");
}
saslProps.put(Sasl.SERVER_AUTH, "true");
if (sessConfMap.containsKey(JdbcConnectionParams.AUTH_PRINCIPAL)) {
transport = KerberosSaslHelper.getKerberosTransport(
sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), host,
socketTransport, saslProps, assumeSubject);
} else {
// If there's a delegation token available then use token based connection
String tokenStr = getClientDelegationToken(sessConfMap);
if (tokenStr != null) {
transport = KerberosSaslHelper.getTokenTransport(tokenStr,
host, socketTransport, saslProps);
} else {
// we are using PLAIN Sasl connection with user/password
String userName = getUserName();
String passwd = getPassword();
// Overlay the SASL transport on top of the base socket transport (SSL or non-SSL)
transport = PlainSaslHelper.getPlainTransport(userName, passwd, socketTransport);
}
}
} else {
// Raw socket connection (non-sasl)
transport = socketTransport;
}
} catch (SaslException e) {
throw new SQLException("Could not create secure connection to "
+ jdbcUriString + ": " + e.getMessage(), " 08S01", e);
}
return transport;
}
SSLConnectionSocketFactory getTwoWaySSLSocketFactory() throws SQLException {
SSLConnectionSocketFactory socketFactory = null;
try {
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(
JdbcConnectionParams.SUNX509_ALGORITHM_STRING,
JdbcConnectionParams.SUNJSSE_ALGORITHM_STRING);
String keyStorePath = sessConfMap.get(JdbcConnectionParams.SSL_KEY_STORE);
String keyStorePassword = sessConfMap.get(JdbcConnectionParams.SSL_KEY_STORE_PASSWORD);
KeyStore sslKeyStore = KeyStore.getInstance(JdbcConnectionParams.SSL_KEY_STORE_TYPE);
if (keyStorePath == null || keyStorePath.isEmpty()) {
throw new IllegalArgumentException(JdbcConnectionParams.SSL_KEY_STORE
+ " Not configured for 2 way SSL connection, keyStorePath param is empty");
}
try (FileInputStream fis = new FileInputStream(keyStorePath)) {
sslKeyStore.load(fis, keyStorePassword.toCharArray());
}
keyManagerFactory.init(sslKeyStore, keyStorePassword.toCharArray());
TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(
JdbcConnectionParams.SUNX509_ALGORITHM_STRING);
String trustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE);
String trustStorePassword = sessConfMap.get(
JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
KeyStore sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
if (trustStorePath == null || trustStorePath.isEmpty()) {
throw new IllegalArgumentException(JdbcConnectionParams.SSL_TRUST_STORE
+ " Not configured for 2 way SSL connection");
}
try (FileInputStream fis = new FileInputStream(trustStorePath)) {
sslTrustStore.load(fis, trustStorePassword.toCharArray());
}
trustManagerFactory.init(sslTrustStore);
SSLContext context = SSLContext.getInstance("TLS");
context.init(keyManagerFactory.getKeyManagers(),
trustManagerFactory.getTrustManagers(), new SecureRandom());
socketFactory = new SSLConnectionSocketFactory(context);
} catch (Exception e) {
throw new SQLException("Error while initializing 2 way ssl socket factory ", e);
}
return socketFactory;
}
// Lookup the delegation token. First in the connection URL, then Configuration
private String getClientDelegationToken(Map<String, String> jdbcConnConf)
throws SQLException {
String tokenStr = null;
if (JdbcConnectionParams.AUTH_TOKEN.equalsIgnoreCase(jdbcConnConf.get(JdbcConnectionParams.AUTH_TYPE))) {
// check delegation token in job conf if any
try {
tokenStr = org.apache.hadoop.hive.shims.Utils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN);
} catch (IOException e) {
throw new SQLException("Error reading token ", e);
}
}
return tokenStr;
}
private void openSession() throws SQLException {
TOpenSessionReq openReq = new TOpenSessionReq();
Map<String, String> openConf = new HashMap<String, String>();
// for remote JDBC client, try to set the conf var using 'set foo=bar'
for (Entry<String, String> hiveConf : connParams.getHiveConfs().entrySet()) {
openConf.put("set:hiveconf:" + hiveConf.getKey(), hiveConf.getValue());
}
// For remote JDBC client, try to set the hive var using 'set hivevar:key=value'
for (Entry<String, String> hiveVar : connParams.getHiveVars().entrySet()) {
openConf.put("set:hivevar:" + hiveVar.getKey(), hiveVar.getValue());
}
// switch the database
openConf.put("use:database", connParams.getDbName());
// set the fetchSize
openConf.put("set:hiveconf:hive.server2.thrift.resultset.default.fetch.size",
Integer.toString(fetchSize));
// set the session configuration
Map<String, String> sessVars = connParams.getSessionVars();
if (sessVars.containsKey(HiveAuthConstants.HS2_PROXY_USER)) {
openConf.put(HiveAuthConstants.HS2_PROXY_USER,
sessVars.get(HiveAuthConstants.HS2_PROXY_USER));
}
openReq.setConfiguration(openConf);
// Store the user name in the open request in case no non-sasl authentication
if (JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))) {
openReq.setUsername(sessConfMap.get(JdbcConnectionParams.AUTH_USER));
openReq.setPassword(sessConfMap.get(JdbcConnectionParams.AUTH_PASSWD));
}
try {
TOpenSessionResp openResp = client.OpenSession(openReq);
// validate connection
Utils.verifySuccess(openResp.getStatus());
if (!supportedProtocols.contains(openResp.getServerProtocolVersion())) {
throw new TException("Unsupported Hive2 protocol");
}
protocol = openResp.getServerProtocolVersion();
sessHandle = openResp.getSessionHandle();
// Update fetchSize if modified by server
String serverFetchSize =
openResp.getConfiguration().get("hive.server2.thrift.resultset.default.fetch.size");
if (serverFetchSize != null) {
fetchSize = Integer.parseInt(serverFetchSize);
}
} catch (TException e) {
LOG.error("Error opening session", e);
throw new SQLException("Could not establish connection to "
+ jdbcUriString + ": " + e.getMessage(), " 08S01", e);
}
isClosed = false;
}
/**
* @return username from sessConfMap
*/
private String getUserName() {
return getSessionValue(JdbcConnectionParams.AUTH_USER, JdbcConnectionParams.ANONYMOUS_USER);
}
/**
* @return password from sessConfMap
*/
private String getPassword() {
return getSessionValue(JdbcConnectionParams.AUTH_PASSWD, JdbcConnectionParams.ANONYMOUS_PASSWD);
}
private boolean isSslConnection() {
return "true".equalsIgnoreCase(sessConfMap.get(JdbcConnectionParams.USE_SSL));
}
private boolean isKerberosAuthMode() {
return !JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))
&& sessConfMap.containsKey(JdbcConnectionParams.AUTH_PRINCIPAL);
}
private boolean isHttpTransportMode() {
String transportMode = sessConfMap.get(JdbcConnectionParams.TRANSPORT_MODE);
if(transportMode != null && (transportMode.equalsIgnoreCase("http"))) {
return true;
}
return false;
}
private boolean isZkDynamicDiscoveryMode() {
return (sessConfMap.get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE) != null)
&& (JdbcConnectionParams.SERVICE_DISCOVERY_MODE_ZOOKEEPER.equalsIgnoreCase(sessConfMap
.get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE)));
}
private void logZkDiscoveryMessage(String message) {
if (isZkDynamicDiscoveryMode()) {
LOG.info(message);
}
}
/**
* Lookup varName in sessConfMap, if its null or empty return the default
* value varDefault
* @param varName
* @param varDefault
* @return
*/
private String getSessionValue(String varName, String varDefault) {
String varValue = sessConfMap.get(varName);
if ((varValue == null) || varValue.isEmpty()) {
varValue = varDefault;
}
return varValue;
}
// copy loginTimeout from driver manager. Thrift timeout needs to be in millis
private void setupLoginTimeout() {
long timeOut = TimeUnit.SECONDS.toMillis(DriverManager.getLoginTimeout());
if (timeOut > Integer.MAX_VALUE) {
loginTimeout = Integer.MAX_VALUE;
} else {
loginTimeout = (int) timeOut;
}
}
public void abort(Executor executor) throws SQLException {
// JDK 1.7
throw new SQLFeatureNotSupportedException("Method not supported");
}
public String getDelegationToken(String owner, String renewer) throws SQLException {
TGetDelegationTokenReq req = new TGetDelegationTokenReq(sessHandle, owner, renewer);
try {
TGetDelegationTokenResp tokenResp = client.GetDelegationToken(req);
Utils.verifySuccess(tokenResp.getStatus());
return tokenResp.getDelegationToken();
} catch (TException e) {
throw new SQLException("Could not retrieve token: " +
e.getMessage(), " 08S01", e);
}
}
public void cancelDelegationToken(String tokenStr) throws SQLException {
TCancelDelegationTokenReq cancelReq = new TCancelDelegationTokenReq(sessHandle, tokenStr);
try {
TCancelDelegationTokenResp cancelResp =
client.CancelDelegationToken(cancelReq);
Utils.verifySuccess(cancelResp.getStatus());
return;
} catch (TException e) {
throw new SQLException("Could not cancel token: " +
e.getMessage(), " 08S01", e);
}
}
public void renewDelegationToken(String tokenStr) throws SQLException {
TRenewDelegationTokenReq cancelReq = new TRenewDelegationTokenReq(sessHandle, tokenStr);
try {
TRenewDelegationTokenResp renewResp =
client.RenewDelegationToken(cancelReq);
Utils.verifySuccess(renewResp.getStatus());
return;
} catch (TException e) {
throw new SQLException("Could not renew token: " +
e.getMessage(), " 08S01", e);
}
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#clearWarnings()
*/
@Override
public void clearWarnings() throws SQLException {
warningChain = null;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#close()
*/
@Override
public void close() throws SQLException {
if (!isClosed) {
TCloseSessionReq closeReq = new TCloseSessionReq(sessHandle);
try {
client.CloseSession(closeReq);
} catch (TException e) {
throw new SQLException("Error while cleaning up the server resources", e);
} finally {
isClosed = true;
if (transport != null) {
transport.close();
}
}
}
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#commit()
*/
@Override
public void commit() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createArrayOf(java.lang.String,
* java.lang.Object[])
*/
@Override
public Array createArrayOf(String arg0, Object[] arg1) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createBlob()
*/
@Override
public Blob createBlob() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createClob()
*/
@Override
public Clob createClob() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createNClob()
*/
@Override
public NClob createNClob() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createSQLXML()
*/
@Override
public SQLXML createSQLXML() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/**
* Creates a Statement object for sending SQL statements to the database.
*
* @throws SQLException
* if a database access error occurs.
* @see java.sql.Connection#createStatement()
*/
@Override
public Statement createStatement() throws SQLException {
if (isClosed) {
throw new SQLException("Can't create Statement, connection is closed");
}
return new HiveStatement(this, client, sessHandle, fetchSize);
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createStatement(int, int)
*/
@Override
public Statement createStatement(int resultSetType, int resultSetConcurrency)
throws SQLException {
if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) {
throw new SQLException("Statement with resultset concurrency " +
resultSetConcurrency + " is not supported", "HYC00"); // Optional feature not implemented
}
if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) {
throw new SQLException("Statement with resultset type " + resultSetType +
" is not supported", "HYC00"); // Optional feature not implemented
}
return new HiveStatement(this, client, sessHandle,
resultSetType == ResultSet.TYPE_SCROLL_INSENSITIVE, fetchSize);
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createStatement(int, int, int)
*/
@Override
public Statement createStatement(int resultSetType, int resultSetConcurrency,
int resultSetHoldability) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#createStruct(java.lang.String, java.lang.Object[])
*/
@Override
public Struct createStruct(String typeName, Object[] attributes)
throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getAutoCommit()
*/
@Override
public boolean getAutoCommit() throws SQLException {
return true;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getCatalog()
*/
@Override
public String getCatalog() throws SQLException {
return "";
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getClientInfo()
*/
@Override
public Properties getClientInfo() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getClientInfo(java.lang.String)
*/
@Override
public String getClientInfo(String name) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getHoldability()
*/
@Override
public int getHoldability() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getMetaData()
*/
@Override
public DatabaseMetaData getMetaData() throws SQLException {
if (isClosed) {
throw new SQLException("Connection is closed");
}
return new HiveDatabaseMetaData(this, client, sessHandle);
}
public int getNetworkTimeout() throws SQLException {
// JDK 1.7
throw new SQLFeatureNotSupportedException("Method not supported");
}
public String getSchema() throws SQLException {
if (isClosed) {
throw new SQLException("Connection is closed");
}
try (Statement stmt = createStatement();
ResultSet res = stmt.executeQuery("SELECT current_database()")) {
if (!res.next()) {
throw new SQLException("Failed to get schema information");
}
return res.getString(1);
}
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getTransactionIsolation()
*/
@Override
public int getTransactionIsolation() throws SQLException {
return Connection.TRANSACTION_NONE;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getTypeMap()
*/
@Override
public Map<String, Class<?>> getTypeMap() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#getWarnings()
*/
@Override
public SQLWarning getWarnings() throws SQLException {
return warningChain;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#isClosed()
*/
@Override
public boolean isClosed() throws SQLException {
return isClosed;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#isReadOnly()
*/
@Override
public boolean isReadOnly() throws SQLException {
return false;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#isValid(int)
*/
@Override
public boolean isValid(int timeout) throws SQLException {
if (timeout < 0) {
throw new SQLException("timeout value was negative");
}
boolean rc = false;
try {
String productName = new HiveDatabaseMetaData(this, client, sessHandle)
.getDatabaseProductName();
rc = true;
} catch (SQLException e) {
// IGNORE
}
return rc;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#nativeSQL(java.lang.String)
*/
@Override
public String nativeSQL(String sql) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareCall(java.lang.String)
*/
@Override
public CallableStatement prepareCall(String sql) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareCall(java.lang.String, int, int)
*/
@Override
public CallableStatement prepareCall(String sql, int resultSetType,
int resultSetConcurrency) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareCall(java.lang.String, int, int, int)
*/
@Override
public CallableStatement prepareCall(String sql, int resultSetType,
int resultSetConcurrency, int resultSetHoldability) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareStatement(java.lang.String)
*/
@Override
public PreparedStatement prepareStatement(String sql) throws SQLException {
return new HivePreparedStatement(this, client, sessHandle, sql);
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareStatement(java.lang.String, int)
*/
@Override
public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys)
throws SQLException {
return new HivePreparedStatement(this, client, sessHandle, sql);
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareStatement(java.lang.String, int[])
*/
@Override
public PreparedStatement prepareStatement(String sql, int[] columnIndexes)
throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareStatement(java.lang.String,
* java.lang.String[])
*/
@Override
public PreparedStatement prepareStatement(String sql, String[] columnNames)
throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareStatement(java.lang.String, int, int)
*/
@Override
public PreparedStatement prepareStatement(String sql, int resultSetType,
int resultSetConcurrency) throws SQLException {
return new HivePreparedStatement(this, client, sessHandle, sql);
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#prepareStatement(java.lang.String, int, int, int)
*/
@Override
public PreparedStatement prepareStatement(String sql, int resultSetType,
int resultSetConcurrency, int resultSetHoldability) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#releaseSavepoint(java.sql.Savepoint)
*/
@Override
public void releaseSavepoint(Savepoint savepoint) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#rollback()
*/
@Override
public void rollback() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#rollback(java.sql.Savepoint)
*/
@Override
public void rollback(Savepoint savepoint) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setAutoCommit(boolean)
*/
@Override
public void setAutoCommit(boolean autoCommit) throws SQLException {
// Per JDBC spec, if the connection is closed a SQLException should be thrown.
if(isClosed) {
throw new SQLException("Connection is closed");
}
// The auto-commit mode is always enabled for this connection. Per JDBC spec,
// if setAutoCommit is called and the auto-commit mode is not changed, the call is a no-op.
if (!autoCommit) {
LOG.warn("Request to set autoCommit to false; Hive does not support autoCommit=false.");
SQLWarning warning = new SQLWarning("Hive does not support autoCommit=false");
if (warningChain == null) warningChain = warning;
else warningChain.setNextWarning(warning);
}
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setCatalog(java.lang.String)
*/
@Override
public void setCatalog(String catalog) throws SQLException {
// Per JDBC spec, if the driver does not support catalogs,
// it will silently ignore this request.
if (isClosed) {
throw new SQLException("Connection is closed");
}
return;
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setClientInfo(java.util.Properties)
*/
@Override
public void setClientInfo(Properties properties)
throws SQLClientInfoException {
// TODO Auto-generated method stub
throw new SQLClientInfoException("Method not supported", null);
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setClientInfo(java.lang.String, java.lang.String)
*/
@Override
public void setClientInfo(String name, String value)
throws SQLClientInfoException {
// TODO Auto-generated method stub
throw new SQLClientInfoException("Method not supported", null);
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setHoldability(int)
*/
@Override
public void setHoldability(int holdability) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException {
// JDK 1.7
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setReadOnly(boolean)
*/
@Override
public void setReadOnly(boolean readOnly) throws SQLException {
// Per JDBC spec, if the connection is closed a SQLException should be thrown.
if (isClosed) {
throw new SQLException("Connection is closed");
}
// Per JDBC spec, the request defines a hint to the driver to enable database optimizations.
// The read-only mode for this connection is disabled and cannot be enabled (isReadOnly always returns false).
// The most correct behavior is to throw only if the request tries to enable the read-only mode.
if(readOnly) {
throw new SQLException("Enabling read-only mode not supported");
}
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setSavepoint()
*/
@Override
public Savepoint setSavepoint() throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setSavepoint(java.lang.String)
*/
@Override
public Savepoint setSavepoint(String name) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
public void setSchema(String schema) throws SQLException {
// JDK 1.7
if (isClosed) {
throw new SQLException("Connection is closed");
}
if (schema == null || schema.isEmpty()) {
throw new SQLException("Schema name is null or empty");
}
Statement stmt = createStatement();
stmt.execute("use " + schema);
stmt.close();
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setTransactionIsolation(int)
*/
@Override
public void setTransactionIsolation(int level) throws SQLException {
// TODO: throw an exception?
}
/*
* (non-Javadoc)
*
* @see java.sql.Connection#setTypeMap(java.util.Map)
*/
@Override
public void setTypeMap(Map<String, Class<?>> map) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Wrapper#isWrapperFor(java.lang.Class)
*/
@Override
public boolean isWrapperFor(Class<?> iface) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
/*
* (non-Javadoc)
*
* @see java.sql.Wrapper#unwrap(java.lang.Class)
*/
@Override
public <T> T unwrap(Class<T> iface) throws SQLException {
// TODO Auto-generated method stub
throw new SQLFeatureNotSupportedException("Method not supported");
}
public TProtocolVersion getProtocol() {
return protocol;
}
public static TCLIService.Iface newSynchronizedClient(
TCLIService.Iface client) {
return (TCLIService.Iface) Proxy.newProxyInstance(
HiveConnection.class.getClassLoader(),
new Class [] { TCLIService.Iface.class },
new SynchronizedHandler(client));
}
private static class SynchronizedHandler implements InvocationHandler {
private final TCLIService.Iface client;
private final ReentrantLock lock = new ReentrantLock(true);
SynchronizedHandler(TCLIService.Iface client) {
this.client = client;
}
@Override
public Object invoke(Object proxy, Method method, Object [] args)
throws Throwable {
try {
lock.lock();
return method.invoke(client, args);
} catch (InvocationTargetException e) {
// all IFace APIs throw TException
if (e.getTargetException() instanceof TException) {
throw (TException)e.getTargetException();
} else {
// should not happen
throw new TException("Error in calling method " + method.getName(),
e.getTargetException());
}
} catch (Exception e) {
throw new TException("Error in calling method " + method.getName(), e);
} finally {
lock.unlock();
}
}
}
}