package com.tesora.dve.sql.parser;
/*
* #%L
* Tesora Inc.
* Database Virtualization Engine
* %%
* Copyright (C) 2011 - 2014 Tesora Inc.
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.tesora.dve.common.PECharsetUtils;
import com.tesora.dve.exceptions.PECodingException;
import com.tesora.dve.exceptions.PEException;
import com.tesora.dve.sql.ParserException;
import com.tesora.dve.sql.ParserException.Pass;
import com.tesora.dve.sql.PlannerStatisticType;
import com.tesora.dve.sql.PlannerStatistics;
import com.tesora.dve.sql.SchemaException;
import com.tesora.dve.sql.node.expression.ExpressionNode;
import com.tesora.dve.sql.schema.SchemaContext;
import com.tesora.dve.sql.schema.cache.CandidateCachedPlan;
import com.tesora.dve.sql.schema.cache.PlanCacheUtils;
import com.tesora.dve.sql.schema.cache.PlanCacheUtils.PlanCacheCallback;
import com.tesora.dve.sql.schema.types.Type;
import com.tesora.dve.sql.statement.CacheableStatement;
import com.tesora.dve.sql.statement.EmptyStatement;
import com.tesora.dve.sql.statement.Statement;
import com.tesora.dve.sql.statement.StatementType;
import com.tesora.dve.sql.statement.dml.DMLStatement;
import com.tesora.dve.sql.statement.session.TransactionStatement;
import com.tesora.dve.sql.transform.execution.ConnectionValuesMap;
import com.tesora.dve.sql.transform.execution.ExecutionPlan;
import com.tesora.dve.sql.transform.execution.RootExecutionPlan;
import com.tesora.dve.sql.util.ListOfPairs;
import com.tesora.dve.sql.util.Pair;
import com.tesora.dve.variables.KnownVariables;
public class InvokeParser {
public static final Logger sqlLogger = Logger.getLogger("sql.logger");
public static final Logger logger = Logger.getLogger(InvokeParser.class);
public static final long defaultLargeInsertThreshold = 1024 * 1024;
public static String parseOneLine(InputState line, ParserOptions opts) throws Exception {
TranslatorUtils utils = new TranslatorUtils(opts, null, line);
PE parser = buildParser(line, utils);
Object firstPass = parser.sql_statements().getTree();
Exception any = utils.buildError();
if (any != null)
throw any;
return utils.displayTree(null, firstPass);
}
public static sql2003Lexer buildLexer(InputState inputStatement, Utils utils) {
ANTLRStringStream input = inputStatement.buildNewStream();
sql2003Lexer lexer = new sql2003Lexer(input);
lexer.setUtils(utils);
return lexer;
}
private static PE buildParser(sql2003Lexer lexer, TranslatorUtils utils) {
// CommonTokenStream tokens = new CommonTokenStream(lexer);
TokenRewriteStream tokens = new TokenRewriteStream(lexer);
PE parser = new PE(tokens);
parser.setUtils(utils);
return parser;
}
private static PE buildParser(InputState line, TranslatorUtils utils) {
return buildParser(buildLexer(line, utils), utils);
}
public static InputState buildInputState(String icmd, SchemaContext pc) {
long maxLen =
KnownVariables.LARGE_INSERT_CUTOFF.getValue(pc == null ? null : pc.getConnection().getVariableSource()).longValue();
if (icmd.length() > maxLen)
return new ContinuationInputState(icmd,maxLen);
return new InitialInputState(icmd);
}
public static ParseResult parse(InputState icmd, ParserOptions opts, SchemaContext pc) throws ParserException {
return parse(icmd,DEFAULT_PARSER_RULE,opts,pc,TranslatorInitCallback.INSTANCE);
}
public static ParseResult parse(InputState icmd, ParserEntryPoint entryPoint, ParserOptions opts, SchemaContext pc, TranslatorInitCallback ticb) throws ParserException {
preparse(pc);
return parse(icmd, entryPoint, opts, pc, Collections.emptyList(),ticb);
}
private static void preparse(SchemaContext pc) throws ParserException {
if (pc != null) {
// before we do anything, figure out if the current user is a
// tenant, and if so, if they have been suspended
pc.getPolicyContext().checkEnabled();
}
}
private static ParseResult parse(InputState input, ParserEntryPoint entry, ParserOptions opts, SchemaContext pc, List<Object> parameters, TranslatorInitCallback cb)
throws ParserException {
// debug log is set only for non tests
if (pc != null) {
pc.setOptions(opts);
pc.setParameters(parameters);
SchemaContext.threadContext.set(pc);
}
Pair<TranslatorUtils, List<Statement>> result = null;
if (pc != null && pc.getIntraStmtState().isUnderLockTable()) {
result = parseFastInsert(pc, opts, input);
}
if (result == null)
result = parse(pc, entry, opts, input, cb);
List<Statement> stmts = result.getSecond();
TranslatorUtils utils = result.getFirst();
if (stmts.isEmpty())
stmts.add(new EmptyStatement(""));
try {
if (!opts.isTSchema())
utils.assignPositions();
} catch (Throwable t) {
throw new ParserException(Pass.SECOND, "Unable to complete parsing for '" + input.describe() + "'", t);
}
for (Statement s : stmts) {
if (logger.isDebugEnabled() && opts.isDebugLog()) {
if (pc != null)
logger.debug(" Statement (" + pc.describeContext() + ") => " + s.getSQL(pc, true, true));
else
logger.debug(" Statement => " + s.getSQL(pc, true, true));
}
}
// break a self referential chain
utils.setContext(null);
return new ParseResult(stmts,utils.getInputState(input));
}
@SuppressWarnings("unchecked")
private static Pair<TranslatorUtils, List<Statement>> parseFastInsert(SchemaContext pc, ParserOptions opts,
InputState icmd) {
// first of all, make sure it's an actual insert statement
if (!icmd.isInsert())
return null;
TranslatorUtils utils = new TranslatorUtils(opts, pc, icmd);
PE parser = buildParser(icmd, utils);
if (pc != null)
pc.setTokenStream(parser.getTokenStream(),icmd.getCommand());
List<Statement> stmts = null;
List<List<ExpressionNode>> continuedInsert = null;
try {
if (icmd.getCurrentPosition() == 0) {
stmts = Collections.singletonList(parser.fast_insert_statement().s);
} else {
continuedInsert = parser.fast_continuation_insert_value_list().l;
utils.popScope();
}
} catch (EnoughException ee) {
return new Pair<TranslatorUtils,List<Statement>>(utils, Collections.singletonList(((Statement)ee.getInsertStatement())));
} catch (Throwable t) {
// basically, just return null and try again
return null;
} finally {
if (pc != null)
pc.clearOrigStmt();
}
ParserException any = utils.buildError();
if (any != null)
return null;
if (stmts == null) {
stmts = new ArrayList<Statement>();
if (continuedInsert == null || continuedInsert.isEmpty()) {
stmts.add(new TransactionStatement(TransactionStatement.Kind.COMMIT));
} else {
stmts.add(utils.buildInsertStatement(continuedInsert,false, TransactionStatement.Kind.COMMIT, null, false));
}
}
return new Pair<TranslatorUtils, List<Statement>>(utils, stmts);
}
public static Type parseType(final SchemaContext pc, final ParserOptions opts, final String typeDescriptor) {
final InputState icmd = buildInputState(typeDescriptor, pc);
final TranslatorUtils utils = new TranslatorUtils(opts, pc, icmd);
final PE parser = buildParser(icmd, utils);
try {
return parser.type_description().type;
} catch (final RecognitionException e) {
throw new PECodingException("Could not parse the type descriptor: '" + typeDescriptor + "'", e);
}
}
public static ExpressionNode parseExpression(SchemaContext pc, String input) {
InputState icmd = buildInputState(input,pc);
ParserOptions opts = ParserOptions.NONE.setDebugLog(true).setResolve().setFailEarly().setActualLiterals();
TranslatorUtils utils = new TranslatorUtils(opts,pc,icmd);
PE parser = buildParser(icmd,utils);
try {
return parser.value_expression().expr;
} catch (Throwable t) {
throw new SchemaException(Pass.PLANNER, "Unable to parser expression '" + input + "'",t);
}
}
@SuppressWarnings("unchecked")
private static Pair<TranslatorUtils, List<Statement>> parse(SchemaContext pc, ParserEntryPoint entry, ParserOptions opts, InputState input, TranslatorInitCallback cb) {
TranslatorUtils utils = new TranslatorUtils(opts, pc, input);
cb.onInit(utils);
PE parser = buildParser(input, utils);
if (pc != null)
pc.setTokenStream(parser.getTokenStream(), input.getCommand());
List<Statement> stmts = null;
List<List<ExpressionNode>> continuedInsert = null;
try {
if (input.getCurrentPosition() == 0) {
stmts = entry.invoke(parser);
// stmts = parser.sql_statements().stmts;
} else {
continuedInsert = parser.continuation_insert_value_list().l;
utils.popScope();
}
} catch (EnoughException ee) {
return new Pair<TranslatorUtils,List<Statement>>(utils, Collections.singletonList(((Statement)ee.getInsertStatement())));
} catch (ParserException pe) {
throw pe;
} catch (Throwable t) {
throw new ParserException(Pass.SECOND, "Unable to parse '" + input.describe() + "'", t);
} finally {
// clear the input string
if (pc != null)
pc.clearOrigStmt();
}
ParserException any = utils.buildError();
if (any != null)
throw any;
if (stmts == null) {
stmts = new ArrayList<Statement>();
if (continuedInsert == null || continuedInsert.isEmpty()) {
stmts.add(new TransactionStatement(TransactionStatement.Kind.COMMIT));
} else {
stmts.add(utils.buildInsertStatement(continuedInsert,false, TransactionStatement.Kind.COMMIT, null, false));
}
}
return new Pair<TranslatorUtils, List<Statement>>(utils, stmts);
}
public static List<Statement> parse(String line, SchemaContext pc) throws ParserException {
return parse(line, pc, Collections.emptyList());
}
public static List<Statement> parse(String line, SchemaContext pc, List<Object> params, ParserOptions options) throws ParserException {
return parse(line,pc,params,options,TranslatorInitCallback.INSTANCE);
}
public static List<Statement> parseTriggerBody(String body, SchemaContext pc, ParserOptions options, TranslatorInitCallback ticb) throws ParserException {
return parse(body,pc,Collections.emptyList(),TRIGGER_PARSER_RULE, options,ticb);
}
public static List<Statement> parse(String line, SchemaContext pc, List<Object> params, ParserOptions options, TranslatorInitCallback ticb) throws ParserException {
return parse(line, pc, params, DEFAULT_PARSER_RULE, options, ticb);
}
public static List<Statement> parse(String line, SchemaContext pc, List<Object> params, ParserEntryPoint entry, ParserOptions opts, TranslatorInitCallback ticb) throws ParserException {
return parse(buildInputState(line,pc), entry, opts, pc, params, ticb).getStatements();
}
public static List<Statement> parse(String line, SchemaContext pc, List<Object> params) throws ParserException {
// the tests were all written to use latin1
ParserOptions options = ParserOptions.NONE.setDebugLog(true).setResolve().setFailEarly();
return parse(line,pc,params,options);
}
public static ParseResult parse(byte[] line, SchemaContext pc, Charset cs)
throws ParserException {
preparse(pc);
ParserOptions options = ParserOptions.NONE.setDebugLog(true).setResolve().setFailEarly();
String lineStr = PECharsetUtils.getString(line, cs, true);
if (lineStr != null) {
lineStr = StringUtils.strip(lineStr, new String(Character.toString(Character.MIN_VALUE)));
return parse(buildInputState(lineStr,pc), options, pc);
}
return parameterizeAndParse(pc, options, line, cs);
}
public static ListOfPairs<Statement,List<Object>> buildParameterizedCommands(String in, Charset cs, SchemaContext pc) {
// note that we are _not_ setting resolve here
ParserOptions options = ParserOptions.NONE.setDebugLog(false);
ParseResult results = parse(buildInputState(in,pc), options, pc);
if (results.hasMore())
throw new ParserException(Pass.FIRST, "Unable to parameterize for invalid chars - input too large");
List<Statement> stmts = parse(buildInputState(in,pc), options, pc).getStatements();
ListOfPairs<Statement,List<Object>> out = new ListOfPairs<Statement,List<Object>>();
for (Statement s : stmts)
try {
out.add(s,s.extractParameters(pc,PECharsetUtils.latin1, cs));
} catch (PEException pe) {
throw new ParserException(Pass.FIRST, "Unable to extract parameters for input stmt '" + in
+ "' in order to handle character set " + cs.name(), pe);
}
return out;
}
private static ParseResult parameterizeAndParse(SchemaContext pc, ParserOptions options, byte[] line,
Charset cs) throws ParserException {
String singleByteEncoded = PECharsetUtils.getString(line, PECharsetUtils.latin1, false);
logParse(pc, singleByteEncoded, null);
ListOfPairs<Statement,List<Object>> parameterized = buildParameterizedCommands(singleByteEncoded, cs, pc);
ArrayList<Statement> out = new ArrayList<Statement>();
for (Pair<Statement, List<Object>> p : parameterized) {
DMLStatement dmls = (DMLStatement) p.getFirst();
String msql = dmls.getSQL(pc, false, true);
List<Object> params = p.getSecond();
byte[] modstmt = msql.getBytes(PECharsetUtils.latin1);
String orig = PECharsetUtils.getString(modstmt, cs, true);
if (orig == null)
throw new ParserException(Pass.FIRST,
"Unable to parameterize SQL statement to handle characters invalid for character set "
+ cs.name());
orig = StringUtils.strip(orig, new String(Character.toString(Character.MIN_VALUE)));
out.addAll(parse(buildInputState(orig,pc), DEFAULT_PARSER_RULE, options, pc, params, TranslatorInitCallback.INSTANCE).getStatements());
}
return new ParseResult(out,null);
}
public static PlanningResult preparePlan(SchemaContext pc, InputState input, ParserOptions options, String pstmtID) throws PEException {
ParseResult pr = parse(input, options, pc);
if (pr.getStatements().size() > 1)
throw new PEException("Cannot prepare more than one statement at a time");
Statement toPrepare = pr.getStatements().get(0);
return Statement.prepare(pc, toPrepare, pc.getBehaviorConfiguration(), pstmtID, input.getCommand());
}
public static PlanningResult buildPlan(SchemaContext pc, InputState input, ParserOptions options, PlanCacheCallback ipcb) throws PEException {
PlanCacheCallback pcb = (ipcb == null ? logCacheCallback : ipcb);
List<ExecutionPlan> plans = null;
ConnectionValuesMap values = null;
boolean tryCache = pc.getSource().canCachePlans(pc) && !pc.getIntraStmtState().isUnderLockTable();
CandidateCachedPlan ccp = null;
if (!pc.getSource().isPlanCacheEmpty() && input.getCommand() != null) {
ccp = PlanCacheUtils.getCachedPlan(pc, input.getCommand(), pcb);
if (ccp.getPlan() != null) {
plans = new ArrayList<ExecutionPlan>();
plans.add(ccp.getPlan());
values = ccp.getValues();
} else if (!ccp.tryCaching()) {
tryCache = false;
}
}
if (plans == null) {
ParseResult pr = parse(input, options, pc);
plans = new ArrayList<ExecutionPlan>();
Statement first = null;
boolean explain = false;
values = new ConnectionValuesMap();
for (Statement s : pr.getStatements()) {
if (first == null) first = s;
PlanningResult builtPlan = null;
if (s.isExplain()) {
explain = true;
builtPlan = buildExplainPlan(pc,s,input.getCommand(),input);
} else {
builtPlan = Statement.getExecutionPlan(pc,s,pc.getBehaviorConfiguration(),input.getCommand(),input);
}
values.take(builtPlan.getValues());
plans.addAll(builtPlan.getPlans());
}
if (pcb != null && input.getCommand() != null && !explain)
pcb.onMiss(input.getCommand());
if (!explain && input.getCommand() != null && plans.size() == 1 && tryCache && first instanceof CacheableStatement) {
PlanCacheUtils.maybeCachePlan(pc,pc.getSource(),(CacheableStatement)first, plans.get(0), input.getCommand(),
ccp == null ? null : ccp.getShrunk());
}
if (ccp != null && ccp.getShrunk() == null && pc.getCurrentDatabase(false) != null) {
if (first instanceof DMLStatement) {
// we have to count this late because the candidate parser can only handle dml statements.
PlannerStatistics.increment(PlannerStatisticType.UNCACHEABLE_CANDIDATE_PARSE);
}
}
return new PlanningResult(plans,values,pr.getInputState(),input.getCommand());
}
// clear the warnings
pc.getConnection().getMessageManager().clear();
return new PlanningResult(plans,values,null,input.getCommand());
}
// if we have an explain - we should try to match to the plan cache
private static PlanningResult buildExplainPlan(SchemaContext sc, Statement s, String origSQL, InputState input) throws PEException {
// if the explain is for raw statistics, or a regular explain - then we can try the plan cache
// otherwise not so much
if (s.getExplain().tryCache()) {
String sql = origSQL.substring(s.getSourceLocation().getPositionInLine());
CandidateCachedPlan ccp = null;
if (!sc.getSource().isPlanCacheEmpty()) {
ccp = PlanCacheUtils.getCachedPlan(sc, sql, null);
if (ccp.getPlan() != null) {
RootExecutionPlan actual = ccp.getPlan();
RootExecutionPlan expep = new RootExecutionPlan(null,actual.getValueManager(), StatementType.EXPLAIN);
expep.getSequence().append(actual.generateExplain(sc,ccp.getValues(),s,sql));
return new PlanningResult(expep,ccp.getValues(),input,origSQL);
}
}
}
// if we're still here - we have to build the old fashioned way
return Statement.getExecutionPlan(sc,s,sc.getBehaviorConfiguration(),origSQL, input);
}
// continuation version
public static PlanningResult buildPlan(SchemaContext pc, InputState stmt) throws PEException {
if (pc !=null )
SchemaContext.threadContext.set(pc);
preparse(pc);
ParserOptions options = ParserOptions.NONE.setDebugLog(true).setResolve();
PlanningResult result = buildPlan(pc, stmt, options, null);
for(ExecutionPlan ep : result.getPlans())
if (ep.isRoot()) {
SqlStatistics.incrementCounter(((RootExecutionPlan)ep).getStatementType());
}
return result;
}
public static PlanningResult buildPlan(SchemaContext pc, byte[] line, Charset cs, String pstmtID)
throws PEException {
boolean isPrepare = (pstmtID != null);
if (pc != null)
SchemaContext.threadContext.set(pc);
preparse(pc);
ParserOptions options = ParserOptions.NONE.setDebugLog(true).setResolve().setFailEarly();
if (isPrepare)
options = options.setPrepare().setActualLiterals();
PlanningResult result = null;
String lineStr = PECharsetUtils.getString(line, cs, true);
if (lineStr == null) {
// bad characters - not a candidate for caching
// if we get this for a prepared stmt, just give up for now
if (isPrepare)
throw new PEException("Invalid prepare request: bad characters");
ParseResult pr = parameterizeAndParse(pc, options, line, cs);
List<Statement> stmts = pr.getStatements();
List<ExecutionPlan> plans = new ArrayList<ExecutionPlan>();
ConnectionValuesMap cvm = new ConnectionValuesMap();
for (Statement s : stmts) {
PlanningResult epr = Statement.getExecutionPlan(pc,s,pc.getBehaviorConfiguration(),lineStr,pr.getInputState());
plans.addAll(epr.getPlans());
cvm.take(epr.getValues());
}
result = new PlanningResult(plans, cvm,pr.getInputState(),lineStr);
} else {
lineStr = StringUtils.strip(lineStr, new String(Character.toString(Character.MIN_VALUE)));
logParse(pc, lineStr, pstmtID);
InputState input = buildInputState(lineStr, pc);
if (isPrepare) {
if (input.getCommand() == null)
throw new PEException("Prepare stmt is too long (greater than " + input.getThreshold() + " characters)");
result = preparePlan(pc, input, options, pstmtID);
} else
result = buildPlan(pc, input, options, null);
}
for(ExecutionPlan ep : result.getPlans())
if (ep.isRoot()) {
SqlStatistics.incrementCounter(((RootExecutionPlan)ep).getStatementType());
}
return result;
}
public static PlanningResult bindPreparedStatement(SchemaContext sc, String stmtID, List<Object> params) throws PEException {
// make sure we clear the options
sc.setOptions(ParserOptions.NONE);
Pair<RootExecutionPlan,ConnectionValuesMap> bound = PlanCacheUtils.bindPreparedStatement(sc, stmtID, params);
SqlStatistics.incrementCounter(bound.getFirst().getStatementType());
return new PlanningResult(Collections.singletonList((ExecutionPlan)bound.getFirst()),bound.getSecond(),null,null);
}
public static PreparePlanningResult reprepareStatement(SchemaContext sc, String rawSQL, String stmtID) throws PEException {
// need to replan
ParserOptions options = ParserOptions.NONE.setDebugLog(true).setResolve().setPrepare().setActualLiterals();
InputState input = buildInputState(rawSQL,sc);
return (PreparePlanningResult) preparePlan(sc, input, options, stmtID);
}
private static void logParse(SchemaContext pc, String inline, String pstmtID) {
boolean isPrepare = (pstmtID != null);
String line = inline;
if (isPrepare)
line = "PREPARE " + line;
if (logger.isDebugEnabled()) {
if (pc != null)
logger.debug("Begin " + (isPrepare ? "" : pstmtID) + " parse:(" + pc.describeContext() + ") " + line);
else
logger.debug("Begin parse:" + line);
}
if (sqlLogger.isDebugEnabled())
logSql(pc,line);
}
public static void enableSqlLogging(boolean enabled) {
sqlLogger.setLevel(enabled ? Level.DEBUG : Level.OFF);
logger.info("Set SQL Logging log level to " + sqlLogger.getLevel().toString());
}
public static boolean isSqlLoggingEnabled() {
return sqlLogger.isDebugEnabled();
}
public static void logSql(SchemaContext pc, String line) {
if (pc != null)
sqlLogger.debug("(" + pc.describeContext() + ") " + line);
else
sqlLogger.debug(line);
}
private static final PlanCacheCallback logCacheCallback = new PlanCacheCallback() {
private final Logger logger = Logger.getLogger(PlanCacheUtils.class);
@Override
public void onHit(String stmt) {
if (logger.isDebugEnabled())
logger.debug("PlanCache hit: " + stmt);
}
@Override
public void onMiss(String stmt) {
if (logger.isDebugEnabled())
logger.debug("PlanCache miss: " + stmt);
}
};
interface ParserEntryPoint {
List<Statement> invoke(PE parser) throws Throwable;
}
private static final ParserEntryPoint DEFAULT_PARSER_RULE = new ParserEntryPoint() {
@Override
public List<Statement> invoke(PE parser) throws Throwable {
return parser.sql_statements().stmts;
}
};
private static final ParserEntryPoint TRIGGER_PARSER_RULE = new ParserEntryPoint() {
@Override
public List<Statement> invoke(PE parser) throws Throwable {
return Collections.singletonList(parser.compound_or_single_statement().s);
}
};
}