/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.hive.ql.optimizer;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Stack;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
/**
* Factory for generating the different node processors used by ConstantPropagate.
*/
public final class ConstantPropagateProcFactory {
protected static final Logger LOG = LoggerFactory.getLogger(ConstantPropagateProcFactory.class.getName());
protected static Set<Class<?>> propagatableUdfs = new HashSet<Class<?>>();
static {
propagatableUdfs.add(GenericUDFOPAnd.class);
};
private ConstantPropagateProcFactory() {
// prevent instantiation
}
/**
* Get ColumnInfo from column expression.
*
* @param rr
* @param desc
* @return
*/
public static ColumnInfo resolveColumn(RowSchema rs,
ExprNodeColumnDesc desc) {
ColumnInfo ci = rs.getColumnInfo(desc.getTabAlias(), desc.getColumn());
if (ci == null) {
ci = rs.getColumnInfo(desc.getColumn());
}
if (ci == null) {
return null;
}
return ci;
}
private static final Set<PrimitiveCategory> unSupportedTypes = ImmutableSet
.<PrimitiveCategory>builder()
.add(PrimitiveCategory.VARCHAR)
.add(PrimitiveCategory.CHAR).build();
private static final Set<PrimitiveCategory> unsafeConversionTypes = ImmutableSet
.<PrimitiveCategory>builder()
.add(PrimitiveCategory.STRING)
.add(PrimitiveCategory.VARCHAR)
.add(PrimitiveCategory.CHAR).build();
/**
* Cast type from expression type to expected type ti.
*
* @param desc constant expression
* @param ti expected type info
* @return cast constant, or null if the type cast failed.
*/
private static ExprNodeConstantDesc typeCast(ExprNodeDesc desc, TypeInfo ti) {
return typeCast(desc, ti, false);
}
/**
* Cast type from expression type to expected type ti.
*
* @param desc constant expression
* @param ti expected type info
* @param performSafeTypeCast when true then don't perform typecast because it could be unsafe (loosing leading zeroes etc.)
* @return cast constant, or null if the type cast failed.
*/
private static ExprNodeConstantDesc typeCast(ExprNodeDesc desc, TypeInfo ti, boolean performSafeTypeCast) {
if (desc instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)desc).getValue()) {
return null;
}
if (!(ti instanceof PrimitiveTypeInfo) || !(desc.getTypeInfo() instanceof PrimitiveTypeInfo)) {
return null;
}
PrimitiveTypeInfo priti = (PrimitiveTypeInfo) ti;
PrimitiveTypeInfo descti = (PrimitiveTypeInfo) desc.getTypeInfo();
if (unSupportedTypes.contains(priti.getPrimitiveCategory())
|| unSupportedTypes.contains(descti.getPrimitiveCategory())) {
// FIXME: support template types. It currently has conflict with ExprNodeConstantDesc
if (LOG.isDebugEnabled()) {
LOG.debug("Unsupported types " + priti + "; " + descti);
}
return null;
}
// We shouldn't cast strings to other types because that can break original data in cases of
// leading zeros or zeros trailing after decimal point.
// Example: "000126" => 126 => "126"
boolean brokenDataTypesCombination = unsafeConversionTypes.contains(
priti.getPrimitiveCategory()) && !unsafeConversionTypes.contains(
descti.getPrimitiveCategory());
if (performSafeTypeCast && brokenDataTypesCombination) {
if (LOG.isDebugEnabled()) {
LOG.debug("Unsupported cast " + priti + "; " + descti);
}
return null;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Casting " + desc + " to type " + ti);
}
ExprNodeConstantDesc c = (ExprNodeConstantDesc) desc;
if (null != c.getFoldedFromVal() && priti.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
// avoid double casting to preserve original string representation of constant.
return new ExprNodeConstantDesc(c.getFoldedFromVal());
}
ObjectInspector origOI =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(desc.getTypeInfo());
ObjectInspector oi =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(ti);
Converter converter = ObjectInspectorConverters.getConverter(origOI, oi);
Object convObj = converter.convert(c.getValue());
// Convert integer related types because converters are not sufficient
if (convObj instanceof Integer) {
switch (priti.getPrimitiveCategory()) {
case BYTE:
convObj = new Byte((byte) (((Integer) convObj).intValue()));
break;
case SHORT:
convObj = new Short((short) ((Integer) convObj).intValue());
break;
case LONG:
convObj = new Long(((Integer) convObj).intValue());
default:
}
}
return new ExprNodeConstantDesc(ti, convObj);
}
public static ExprNodeDesc foldExpr(ExprNodeGenericFuncDesc funcDesc) {
GenericUDF udf = funcDesc.getGenericUDF();
if (!isDeterministicUdf(udf, funcDesc.getChildren())) {
return funcDesc;
}
return evaluateFunction(funcDesc.getGenericUDF(),funcDesc.getChildren(), funcDesc.getChildren());
}
/**
* Fold input expression desc.
*
* @param desc folding expression
* @param constants current propagated constant map
* @param cppCtx
* @param op processing operator
* @param propagate if true, assignment expressions will be added to constants.
* @return fold expression
* @throws UDFArgumentException
*/
private static ExprNodeDesc foldExpr(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
boolean propagate) throws UDFArgumentException {
if (cppCtx.getConstantPropagateOption() == ConstantPropagateOption.SHORTCUT) {
return foldExprShortcut(desc, constants, cppCtx, op, tag, propagate);
}
return foldExprFull(desc, constants, cppCtx, op, tag, propagate);
}
/**
* Combines the logical not() operator with the child operator if possible.
* @param desc the expression to be evaluated
* @return the new expression to be replaced
* @throws UDFArgumentException
*/
private static ExprNodeDesc foldNegative(ExprNodeDesc desc) throws UDFArgumentException {
if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
GenericUDF udf = funcDesc.getGenericUDF();
if (udf instanceof GenericUDFOPNot) {
ExprNodeDesc child = funcDesc.getChildren().get(0);
if (child instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc childDesc = (ExprNodeGenericFuncDesc)child;
GenericUDF childUDF = childDesc.getGenericUDF();
List<ExprNodeDesc> grandChildren = child.getChildren();
if (childUDF instanceof GenericUDFBaseCompare ||
childUDF instanceof GenericUDFOPNull ||
childUDF instanceof GenericUDFOPNotNull) {
List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
for(ExprNodeDesc grandChild : grandChildren) {
newGrandChildren.add(foldNegative(grandChild));
}
return ExprNodeGenericFuncDesc.newInstance(
childUDF.negative(),
newGrandChildren);
} else if (childUDF instanceof GenericUDFOPAnd ||
childUDF instanceof GenericUDFOPOr) {
List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
for(ExprNodeDesc grandChild : grandChildren) {
newGrandChildren.add(foldNegative(
ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(),
Arrays.asList(grandChild))));
}
return ExprNodeGenericFuncDesc.newInstance(
childUDF.negative(),
newGrandChildren);
}else if (childUDF instanceof GenericUDFOPNot) {
return foldNegative(child.getChildren().get(0));
} else {
// For operator like if() that cannot be handled, leave not() as it
// is and continue processing the children
List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
for(ExprNodeDesc grandChild : grandChildren) {
newGrandChildren.add(foldNegative(grandChild));
}
childDesc.setChildren(newGrandChildren);
return funcDesc;
}
}
}
}
return desc;
}
/**
* Fold input expression desc, only performing short-cutting.
*
* Unnecessary AND/OR operations involving a constant true/false value will be eliminated.
*
* @param desc folding expression
* @param constants current propagated constant map
* @param cppCtx
* @param op processing operator
* @param propagate if true, assignment expressions will be added to constants.
* @return fold expression
* @throws UDFArgumentException
*/
private static ExprNodeDesc foldExprShortcut(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
boolean propagate) throws UDFArgumentException {
// Combine NOT operator with the child operator. Otherwise, the following optimization
// from bottom up could lead to incorrect result, such as not(x > 3 and x is not null),
// should not be optimized to not(x > 3), but (x <=3 or x is null).
desc = foldNegative(desc);
if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
GenericUDF udf = funcDesc.getGenericUDF();
boolean propagateNext = propagate && propagatableUdfs.contains(udf.getClass());
List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc childExpr : desc.getChildren()) {
newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext));
}
// Don't evaluate nondeterministic function since the value can only calculate during runtime.
if (!isDeterministicUdf(udf, newExprs)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evaluate immediately.");
}
((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
return desc;
}
// Check if the function can be short cut.
ExprNodeDesc shortcut = shortcutFunction(udf, newExprs, op);
if (shortcut != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Folding expression:" + desc + " -> " + shortcut);
}
return shortcut;
}
((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
}
return desc;
}
/**
* Fold input expression desc.
*
* This function recursively checks if any subexpression of a specified expression
* can be evaluated to be constant and replaces such subexpression with the constant.
* If the expression is a deterministic UDF and all the subexpressions are constants,
* the value will be calculated immediately (during compilation time vs. runtime).
* e.g.:
* concat(year, month) => 200112 for year=2001, month=12 since concat is deterministic UDF
* unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nondeterministic UDF
* @param desc folding expression
* @param constants current propagated constant map
* @param cppCtx
* @param op processing operator
* @param propagate if true, assignment expressions will be added to constants.
* @return fold expression
* @throws UDFArgumentException
*/
private static ExprNodeDesc foldExprFull(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
boolean propagate) throws UDFArgumentException {
// Combine NOT operator with the child operator. Otherwise, the following optimization
// from bottom up could lead to incorrect result, such as not(x > 3 and x is not null),
// should not be optimized to not(x > 3), but (x <=3 or x is null).
desc = foldNegative(desc);
if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
GenericUDF udf = funcDesc.getGenericUDF();
boolean propagateNext = propagate && propagatableUdfs.contains(udf.getClass());
List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc childExpr : desc.getChildren()) {
newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext));
}
// Don't evaluate nondeterministic function since the value can only calculate during runtime.
if (!isDeterministicUdf(udf, newExprs)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evaluate immediately.");
}
((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
return desc;
} else {
// If all child expressions of deterministic function are constants, evaluate such UDF immediately
ExprNodeDesc constant = evaluateFunction(udf, newExprs, desc.getChildren());
if (constant != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Folding expression:" + desc + " -> " + constant);
}
return constant;
} else {
// Check if the function can be short cut.
ExprNodeDesc shortcut = shortcutFunction(udf, newExprs, op);
if (shortcut != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Folding expression:" + desc + " -> " + shortcut);
}
return shortcut;
}
((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
}
// If in some selected binary operators (=, is null, etc), one of the
// expressions are
// constant, add them to colToConstants as half-deterministic columns.
if (propagate) {
propagate(udf, newExprs, op.getSchema(), constants);
}
}
return desc;
} else if (desc instanceof ExprNodeColumnDesc) {
if (op.getParentOperators() == null || op.getParentOperators().isEmpty()) {
return desc;
}
Operator<? extends Serializable> parent = op.getParentOperators().get(tag);
ExprNodeDesc col = evaluateColumn((ExprNodeColumnDesc) desc, cppCtx, parent);
if (col != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Folding expression:" + desc + " -> " + col);
}
return col;
}
}
return desc;
}
private static boolean isDeterministicUdf(GenericUDF udf, List<ExprNodeDesc> children) {
UDFType udfType = udf.getClass().getAnnotation(UDFType.class);
if (udf instanceof GenericUDFBridge) {
udfType = ((GenericUDFBridge) udf).getUdfClass().getAnnotation(UDFType.class);
}
if (udfType.deterministic() == false) {
if (udf.getClass().equals(GenericUDFUnixTimeStamp.class)
&& children != null && children.size() > 0) {
// unix_timestamp is polymorphic (ignore class annotations)
return true;
}
return false;
}
// If udf is requiring additional jars, we can't determine the result in
// compile time.
String[] files;
String[] jars;
if (udf instanceof GenericUDFBridge) {
GenericUDFBridge bridge = (GenericUDFBridge) udf;
String udfClassName = bridge.getUdfClassName();
try {
UDF udfInternal =
(UDF) Class.forName(bridge.getUdfClassName(), true, Utilities.getSessionSpecifiedClassLoader())
.newInstance();
files = udfInternal.getRequiredFiles();
jars = udfInternal.getRequiredJars();
} catch (Exception e) {
LOG.error("The UDF implementation class '" + udfClassName
+ "' is not present in the class path");
return false;
}
} else {
files = udf.getRequiredFiles();
jars = udf.getRequiredJars();
}
if (files != null || jars != null) {
return false;
}
return true;
}
/**
* Propagate assignment expression, adding an entry into constant map constants.
*
* @param udf expression UDF, currently only 2 UDFs are supported: '=' and 'is null'.
* @param newExprs child expressions (parameters).
* @param cppCtx
* @param op
* @param constants
*/
private static void propagate(GenericUDF udf, List<ExprNodeDesc> newExprs, RowSchema rs,
Map<ColumnInfo, ExprNodeDesc> constants) {
if (udf instanceof GenericUDFOPEqual) {
ExprNodeDesc lOperand = newExprs.get(0);
ExprNodeDesc rOperand = newExprs.get(1);
ExprNodeConstantDesc v;
if (lOperand instanceof ExprNodeConstantDesc) {
v = (ExprNodeConstantDesc) lOperand;
} else if (rOperand instanceof ExprNodeConstantDesc) {
v = (ExprNodeConstantDesc) rOperand;
} else {
// we need a constant on one side.
return;
}
// If both sides are constants, there is nothing to propagate
ExprNodeColumnDesc c = ExprNodeDescUtils.getColumnExpr(lOperand);
if (null == c) {
c = ExprNodeDescUtils.getColumnExpr(rOperand);
}
if (null == c) {
// we need a column expression on other side.
return;
}
ColumnInfo ci = resolveColumn(rs, c);
if (ci != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it.");
}
if (!v.getTypeInfo().equals(ci.getType())) {
v = typeCast(v, ci.getType(), true);
}
if (v != null) {
constants.put(ci, v);
}
}
} else if (udf instanceof GenericUDFOPNull) {
ExprNodeDesc operand = newExprs.get(0);
if (operand instanceof ExprNodeColumnDesc) {
if (LOG.isDebugEnabled()) {
LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it.");
}
ExprNodeColumnDesc c = (ExprNodeColumnDesc) operand;
ColumnInfo ci = resolveColumn(rs, c);
if (ci != null) {
constants.put(ci, new ExprNodeConstantDesc(ci.getType(), null));
}
}
}
}
private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs,
Operator<? extends Serializable> op) throws UDFArgumentException {
if (udf instanceof GenericUDFOPEqual) {
assert newExprs.size() == 2;
boolean foundUDFInFirst = false;
ExprNodeGenericFuncDesc caseOrWhenexpr = null;
if (newExprs.get(0) instanceof ExprNodeGenericFuncDesc) {
caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(0);
if (caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase) {
foundUDFInFirst = true;
}
}
if (!foundUDFInFirst && newExprs.get(1) instanceof ExprNodeGenericFuncDesc) {
caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(1);
if (!(caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase)) {
return null;
}
}
if (null == caseOrWhenexpr) {
// we didn't find case or when udf
return null;
}
GenericUDF childUDF = caseOrWhenexpr.getGenericUDF();
List<ExprNodeDesc> children = caseOrWhenexpr.getChildren();
int i;
if (childUDF instanceof GenericUDFWhen) {
for (i = 1; i < children.size(); i+=2) {
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
}
if(children.size() % 2 == 1) {
i = children.size()-1;
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
}
// after constant folding of child expression the return type of UDFWhen might have changed,
// so recreate the expression
ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF,
caseOrWhenexpr.getFuncText(), children);
return newCaseOrWhenExpr;
} else if (childUDF instanceof GenericUDFCase) {
for (i = 2; i < children.size(); i+=2) {
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
}
if(children.size() % 2 == 0) {
i = children.size()-1;
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
}
// after constant folding of child expression the return type of UDFCase might have changed,
// so recreate the expression
ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF,
caseOrWhenexpr.getFuncText(), children);
return newCaseOrWhenExpr;
} else {
// cant happen
return null;
}
}
if (udf instanceof GenericUDFOPAnd) {
final BitSet positionsToRemove = new BitSet();
final List<ExprNodeDesc> notNullExprs = new ArrayList<ExprNodeDesc>();
final List<Integer> notNullExprsPositions = new ArrayList<Integer>();
final List<ExprNodeDesc> compareExprs = new ArrayList<ExprNodeDesc>();
for (int i = 0; i < newExprs.size(); i++) {
ExprNodeDesc childExpr = newExprs.get(i);
if (childExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
if (Boolean.TRUE.equals(c.getValue())) {
// if true, prune it
positionsToRemove.set(i);
} else {
// if false, return false
return childExpr;
}
} else if (childExpr instanceof ExprNodeGenericFuncDesc &&
((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFOPNotNull &&
childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) {
notNullExprs.add(childExpr.getChildren().get(0));
notNullExprsPositions.add(i);
} else if (childExpr instanceof ExprNodeGenericFuncDesc &&
((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFBaseCompare &&
!(((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFOPNotEqual) &&
childExpr.getChildren().size() == 2) {
// Try to fold (key <op> 86) and (key is not null) to (key <op> 86)
// where <op> can be "=", ">=", "<=", ">", "<".
// Note: (key <> 86) and (key is not null) cannot be folded
ExprNodeColumnDesc colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(0));
if (null == colDesc) {
colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(1));
}
if (colDesc != null) {
compareExprs.add(colDesc);
}
}
}
// Try to fold (key = 86) and (key is not null) to (key = 86)
for (int i = 0; i < notNullExprs.size(); i++) {
for (ExprNodeDesc other : compareExprs) {
if (notNullExprs.get(i).isSame(other)) {
positionsToRemove.set(notNullExprsPositions.get(i));
break;
}
}
}
// Remove unnecessary expressions
int pos = 0;
int removed = 0;
while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
newExprs.remove(pos - removed);
pos++;
removed++;
}
if (newExprs.size() == 0) {
return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.TRUE);
}
if (newExprs.size() == 1) {
return newExprs.get(0);
}
}
if (udf instanceof GenericUDFOPOr) {
final BitSet positionsToRemove = new BitSet();
for (int i = 0; i < newExprs.size(); i++) {
ExprNodeDesc childExpr = newExprs.get(i);
if (childExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr;
if (Boolean.FALSE.equals(c.getValue())) {
// if false, prune it
positionsToRemove.set(i);
} else
if (Boolean.TRUE.equals(c.getValue())) {
// if true return true
return childExpr;
}
}
}
int pos = 0;
int removed = 0;
while ((pos = positionsToRemove.nextSetBit(pos)) != -1) {
newExprs.remove(pos - removed);
pos++;
removed++;
}
if (newExprs.size() == 0) {
return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.FALSE);
}
if (newExprs.size() == 1) {
return newExprs.get(0);
}
}
if (udf instanceof GenericUDFWhen) {
if (!(newExprs.size() == 2 || newExprs.size() == 3)) {
// In general, when can have unlimited # of branches,
// we currently only handle either 1 or 2 branch.
return null;
}
ExprNodeDesc thenExpr = newExprs.get(1);
ExprNodeDesc elseExpr = newExprs.size() == 3 ? newExprs.get(2) :
new ExprNodeConstantDesc(newExprs.get(1).getTypeInfo(),null);
ExprNodeDesc whenExpr = newExprs.get(0);
if (whenExpr instanceof ExprNodeConstantDesc) {
Boolean whenVal = (Boolean)((ExprNodeConstantDesc) whenExpr).getValue();
return (whenVal == null || Boolean.FALSE.equals(whenVal)) ? elseExpr : thenExpr;
}
if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
Object thenVal = constThen.getValue();
Object elseVal = constElse.getValue();
if (thenVal == null) {
if (elseVal == null) {
// both branches are null.
return thenExpr;
} else if (op instanceof FilterOperator) {
// we can still fold, since here null is equivalent to false.
return Boolean.TRUE.equals(elseVal) ?
ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), newExprs.subList(0, 1)) : Boolean.FALSE.equals(elseVal) ?
elseExpr : null;
} else {
// can't do much, expression is not in context of filter, so we can't treat null as equivalent to false here.
return null;
}
} else if (elseVal == null && op instanceof FilterOperator) {
return Boolean.TRUE.equals(thenVal) ? whenExpr : Boolean.FALSE.equals(thenVal) ? thenExpr : null;
} else if(thenVal.equals(elseVal)){
return thenExpr;
} else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
List<ExprNodeDesc> children = new ArrayList<>();
children.add(whenExpr);
children.add(new ExprNodeConstantDesc(false));
ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(),
children);
if (Boolean.TRUE.equals(thenVal)) {
return func;
} else {
List<ExprNodeDesc> exprs = new ArrayList<>();
exprs.add(func);
return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
}
} else {
return null;
}
}
}
if (udf instanceof GenericUDFCase) {
// HIVE-9644 Attempt to fold expression like :
// where (case ss_sold_date when '1998-01-01' then 1=1 else null=1 end);
// where ss_sold_date= '1998-01-01' ;
if (!(newExprs.size() == 3 || newExprs.size() == 4)) {
// In general case can have unlimited # of branches,
// we currently only handle either 1 or 2 branch.
return null;
}
ExprNodeDesc thenExpr = newExprs.get(2);
ExprNodeDesc elseExpr = newExprs.size() == 4 ? newExprs.get(3) :
new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(),null);
if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr;
ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr;
Object thenVal = constThen.getValue();
Object elseVal = constElse.getValue();
if (thenVal == null) {
if (null == elseVal) {
return thenExpr;
} else if (op instanceof FilterOperator) {
return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNotEqual(), newExprs.subList(0, 2)) :
Boolean.FALSE.equals(elseVal) ? elseExpr : null;
} else {
return null;
}
} else if (null == elseVal && op instanceof FilterOperator) {
return Boolean.TRUE.equals(thenVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2)) :
Boolean.FALSE.equals(thenVal) ? thenExpr : null;
} else if(thenVal.equals(elseVal)){
return thenExpr;
} else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
ExprNodeGenericFuncDesc equal = ExprNodeGenericFuncDesc.newInstance(
new GenericUDFOPEqual(), newExprs.subList(0, 2));
List<ExprNodeDesc> children = new ArrayList<>();
children.add(equal);
children.add(new ExprNodeConstantDesc(false));
ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(),
children);
if (Boolean.TRUE.equals(thenVal)) {
return func;
} else {
List<ExprNodeDesc> exprs = new ArrayList<>();
exprs.add(func);
return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs);
}
} else {
return null;
}
}
}
if (udf instanceof GenericUDFUnixTimeStamp) {
if (newExprs.size() >= 1) {
// unix_timestamp(args) -> to_unix_timestamp(args)
return ExprNodeGenericFuncDesc.newInstance(new GenericUDFToUnixTimeStamp(), newExprs);
}
}
return null;
}
/**
* Evaluate column, replace the deterministic columns with constants if possible
*
* @param desc
* @param ctx
* @param op
* @param colToConstants
* @return
*/
private static ExprNodeDesc evaluateColumn(ExprNodeColumnDesc desc,
ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> parent) {
RowSchema rs = parent.getSchema();
ColumnInfo ci = rs.getColumnInfo(desc.getColumn());
if (ci == null) {
if (LOG.isErrorEnabled()) {
LOG.error("Reverse look up of column " + desc + " error!");
}
ci = rs.getColumnInfo(desc.getTabAlias(), desc.getColumn());
}
if (ci == null) {
if (LOG.isErrorEnabled()) {
LOG.error("Can't resolve " + desc.getTabAlias() + "." + desc.getColumn());
}
return null;
}
ExprNodeDesc constant = null;
// Additional work for union operator, see union27.q
if (ci.getAlias() == null) {
for (Entry<ColumnInfo, ExprNodeDesc> e : cppCtx.getOpToConstantExprs().get(parent).entrySet()) {
if (e.getKey().getInternalName().equals(ci.getInternalName())) {
constant = e.getValue();
break;
}
}
} else {
constant = cppCtx.getOpToConstantExprs().get(parent).get(ci);
}
if (constant != null) {
if (constant instanceof ExprNodeConstantDesc
&& !constant.getTypeInfo().equals(desc.getTypeInfo())) {
return typeCast(constant, desc.getTypeInfo());
}
return constant;
} else {
return null;
}
}
/**
* Evaluate UDF
*
* @param udf UDF object
* @param exprs
* @param oldExprs
* @return null if expression cannot be evaluated (not all parameters are constants). Or evaluated
* ExprNodeConstantDesc if possible.
* @throws HiveException
*/
private static ExprNodeDesc evaluateFunction(GenericUDF udf, List<ExprNodeDesc> exprs,
List<ExprNodeDesc> oldExprs) {
DeferredJavaObject[] arguments = new DeferredJavaObject[exprs.size()];
ObjectInspector[] argois = new ObjectInspector[exprs.size()];
for (int i = 0; i < exprs.size(); i++) {
ExprNodeDesc desc = exprs.get(i);
if (desc instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc constant = (ExprNodeConstantDesc) exprs.get(i);
if (!constant.getTypeInfo().equals(oldExprs.get(i).getTypeInfo())) {
constant = typeCast(constant, oldExprs.get(i).getTypeInfo());
if (constant == null) {
return null;
}
}
if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) {
// nested complex types cannot be folded cleanly
return null;
}
Object value = constant.getValue();
PrimitiveTypeInfo pti = (PrimitiveTypeInfo) constant.getTypeInfo();
Object writableValue = null == value ? value :
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti)
.getPrimitiveWritableObject(value);
arguments[i] = new DeferredJavaObject(writableValue);
argois[i] =
ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(),
writableValue);
} else if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc)desc);
if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) {
return null;
}
ExprNodeConstantDesc constant = (ExprNodeConstantDesc) evaluatedFn;
if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) {
// nested complex types cannot be folded cleanly
return null;
}
Object writableValue = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
(PrimitiveTypeInfo) constant.getTypeInfo()).getPrimitiveWritableObject(constant.getValue());
arguments[i] = new DeferredJavaObject(writableValue);
argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue);
} else {
return null;
}
}
try {
ObjectInspector oi = udf.initialize(argois);
Object o = udf.evaluate(arguments);
if (LOG.isDebugEnabled()) {
LOG.debug(udf.getClass().getName() + "(" + exprs + ")=" + o);
}
if (o == null) {
return new ExprNodeConstantDesc(
TypeInfoUtils.getTypeInfoFromObjectInspector(oi), o);
}
Class<?> clz = o.getClass();
if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(clz)) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
TypeInfo typeInfo = poi.getTypeInfo();
o = poi.getPrimitiveJavaObject(o);
if (typeInfo.getTypeName().contains(serdeConstants.DECIMAL_TYPE_NAME)
|| typeInfo.getTypeName()
.contains(serdeConstants.VARCHAR_TYPE_NAME)
|| typeInfo.getTypeName().contains(serdeConstants.CHAR_TYPE_NAME)) {
return new ExprNodeConstantDesc(typeInfo, o);
}
} else if (udf instanceof GenericUDFStruct
&& oi instanceof StandardConstantStructObjectInspector) {
// do not fold named_struct, only struct()
ConstantObjectInspector coi = (ConstantObjectInspector) oi;
TypeInfo structType = TypeInfoUtils.getTypeInfoFromObjectInspector(coi);
return new ExprNodeConstantDesc(structType,
ObjectInspectorUtils.copyToStandardJavaObject(o, coi));
} else if (!PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(clz)) {
if (LOG.isErrorEnabled()) {
LOG.error("Unable to evaluate " + udf
+ ". Return value unrecoginizable.");
}
return null;
} else {
// fall through
}
String constStr = null;
if (arguments.length == 1 && FunctionRegistry.isOpCast(udf)) {
// remember original string representation of constant.
constStr = arguments[0].get().toString();
}
return new ExprNodeConstantDesc(o).setFoldedFromVal(constStr);
} catch (HiveException e) {
LOG.error("Evaluation function " + udf.getClass()
+ " failed in Constant Propagation Optimizer.");
throw new RuntimeException(e);
}
}
/**
* Change operator row schema, replace column with constant if it is.
*
* @param op
* @param constants
* @throws SemanticException
*/
private static void foldOperator(Operator<? extends Serializable> op,
ConstantPropagateProcCtx cppCtx) throws SemanticException {
RowSchema schema = op.getSchema();
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getOpToConstantExprs().get(op);
if (schema != null && schema.getSignature() != null) {
for (ColumnInfo col : schema.getSignature()) {
ExprNodeDesc constant = constants.get(col);
if (constant != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Replacing column " + col + " with constant " + constant + " in " + op);
}
if (!col.getType().equals(constant.getTypeInfo())) {
constant = typeCast(constant, col.getType());
}
if (constant != null) {
col.setObjectinspector(constant.getWritableObjectInspector());
}
}
}
}
Map<String, ExprNodeDesc> colExprMap = op.getColumnExprMap();
if (colExprMap != null) {
for (Entry<ColumnInfo, ExprNodeDesc> e : constants.entrySet()) {
String internalName = e.getKey().getInternalName();
if (colExprMap.containsKey(internalName)) {
colExprMap.put(internalName, e.getValue());
}
}
}
}
/**
* Node Processor for Constant Propagation on Filter Operators. The processor is to fold
* conditional expressions and extract assignment expressions and propagate them.
*/
public static class ConstantPropagateFilterProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
FilterOperator op = (FilterOperator) nd;
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, constants);
ExprNodeDesc condn = op.getConf().getPredicate();
if (LOG.isDebugEnabled()) {
LOG.debug("Old filter FIL[" + op.getIdentifier() + "] conditions:" + condn.getExprString());
}
ExprNodeDesc newCondn = foldExpr(condn, constants, cppCtx, op, 0, true);
if (newCondn instanceof ExprNodeConstantDesc) {
ExprNodeConstantDesc c = (ExprNodeConstantDesc) newCondn;
if (Boolean.TRUE.equals(c.getValue())) {
cppCtx.addOpToDelete(op);
if (LOG.isDebugEnabled()) {
LOG.debug("Filter expression " + condn + " holds true. Will delete it.");
}
} else if (Boolean.FALSE.equals(c.getValue())) {
if (LOG.isWarnEnabled()) {
LOG.warn("Filter expression " + condn + " holds false!");
}
}
}
if (newCondn instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc)newCondn).getValue() == null) {
// where null is same as where false
newCondn = new ExprNodeConstantDesc(Boolean.FALSE);
}
if (LOG.isDebugEnabled()) {
LOG.debug("New filter FIL[" + op.getIdentifier() + "] conditions:" + newCondn.getExprString());
}
// merge it with the downstream col list
op.getConf().setPredicate(newCondn);
foldOperator(op, cppCtx);
return null;
}
}
/**
* Factory method to get the ConstantPropagateFilterProc class.
*
* @return ConstantPropagateFilterProc
*/
public static ConstantPropagateFilterProc getFilterProc() {
return new ConstantPropagateFilterProc();
}
/**
* Node Processor for Constant Propagate for Group By Operators.
*/
public static class ConstantPropagateGroupByProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
GroupByOperator op = (GroupByOperator) nd;
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Map<ColumnInfo, ExprNodeDesc> colToConstants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, colToConstants);
RowSchema rs = op.getSchema();
if (op.getColumnExprMap() != null && rs != null) {
for (ColumnInfo colInfo : rs.getSignature()) {
if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) {
ExprNodeDesc expr = op.getColumnExprMap().get(colInfo.getInternalName());
if (expr instanceof ExprNodeConstantDesc) {
colToConstants.put(colInfo, expr);
}
}
}
}
if (colToConstants.isEmpty()) {
return null;
}
GroupByDesc conf = op.getConf();
ArrayList<ExprNodeDesc> keys = conf.getKeys();
for (int i = 0; i < keys.size(); i++) {
ExprNodeDesc key = keys.get(i);
ExprNodeDesc newkey = foldExpr(key, colToConstants, cppCtx, op, 0, false);
keys.set(i, newkey);
}
foldOperator(op, cppCtx);
return null;
}
}
/**
* Factory method to get the ConstantPropagateGroupByProc class.
*
* @return ConstantPropagateGroupByProc
*/
public static ConstantPropagateGroupByProc getGroupByProc() {
return new ConstantPropagateGroupByProc();
}
/**
* The Default Node Processor for Constant Propagation.
*/
public static class ConstantPropagateDefaultProc implements NodeProcessor {
@Override
@SuppressWarnings("unchecked")
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Operator<? extends Serializable> op = (Operator<? extends Serializable>) nd;
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, constants);
RowSchema rs = op.getSchema();
if (op.getColumnExprMap() != null && rs != null) {
for (ColumnInfo colInfo : rs.getSignature()) {
if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) {
ExprNodeDesc expr = op.getColumnExprMap().get(colInfo.getInternalName());
if (expr instanceof ExprNodeConstantDesc) {
constants.put(colInfo, expr);
}
}
}
}
if (constants.isEmpty()) {
return null;
}
foldOperator(op, cppCtx);
return null;
}
}
/**
* Factory method to get the ConstantPropagateDefaultProc class.
*
* @return ConstantPropagateDefaultProc
*/
public static ConstantPropagateDefaultProc getDefaultProc() {
return new ConstantPropagateDefaultProc();
}
/**
* The Node Processor for Constant Propagation for Select Operators.
*/
public static class ConstantPropagateSelectProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
SelectOperator op = (SelectOperator) nd;
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, constants);
foldOperator(op, cppCtx);
List<ExprNodeDesc> colList = op.getConf().getColList();
List<String> columnNames = op.getConf().getOutputColumnNames();
Map<String, ExprNodeDesc> columnExprMap = op.getColumnExprMap();
if (colList != null) {
for (int i = 0; i < colList.size(); i++) {
ExprNodeDesc newCol = foldExpr(colList.get(i), constants, cppCtx, op, 0, false);
if (!(colList.get(i) instanceof ExprNodeConstantDesc) && newCol instanceof ExprNodeConstantDesc) {
// Lets try to store original column name, if this column got folded
// This is useful for optimizations like GroupByOptimizer
String colName = colList.get(i).getExprString();
if (HiveConf.getPositionFromInternalName(colName) == -1) {
// if its not an internal name, this is what we want.
((ExprNodeConstantDesc)newCol).setFoldedFromCol(colName);
} else {
// If it was internal column, lets try to get name from columnExprMap
ExprNodeDesc desc = columnExprMap.get(colName);
if (desc instanceof ExprNodeConstantDesc) {
((ExprNodeConstantDesc)newCol).setFoldedFromCol(((ExprNodeConstantDesc)desc).getFoldedFromCol());
}
}
}
colList.set(i, newCol);
if (newCol instanceof ExprNodeConstantDesc && op.getSchema() != null) {
ColumnInfo colInfo = op.getSchema().getSignature().get(i);
if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) {
constants.put(colInfo, newCol);
}
}
if (columnExprMap != null) {
columnExprMap.put(columnNames.get(i), newCol);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("New column list:(" + StringUtils.join(colList, " ") + ")");
}
}
return null;
}
}
/**
* The Factory method to get the ConstantPropagateSelectProc class.
*
* @return ConstantPropagateSelectProc
*/
public static ConstantPropagateSelectProc getSelectProc() {
return new ConstantPropagateSelectProc();
}
/**
* The Node Processor for constant propagation for FileSink Operators. In addition to constant
* propagation, this processor also prunes dynamic partitions to static partitions if possible.
*/
public static class ConstantPropagateFileSinkProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
FileSinkOperator op = (FileSinkOperator) nd;
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, constants);
if (constants.isEmpty()) {
return null;
}
FileSinkDesc fsdesc = op.getConf();
DynamicPartitionCtx dpCtx = fsdesc.getDynPartCtx();
if (dpCtx != null) {
// Assume only 1 parent for FS operator
Operator<? extends Serializable> parent = op.getParentOperators().get(0);
Map<ColumnInfo, ExprNodeDesc> parentConstants = cppCtx.getPropagatedConstants(parent);
RowSchema rs = parent.getSchema();
boolean allConstant = true;
int dpColStartIdx = Utilities.getDPColOffset(fsdesc);
List<ColumnInfo> colInfos = rs.getSignature();
for (int i = dpColStartIdx; i < colInfos.size(); i++) {
ColumnInfo ci = colInfos.get(i);
if (parentConstants.get(ci) == null) {
allConstant = false;
break;
}
}
if (allConstant) {
pruneDP(fsdesc);
}
}
foldOperator(op, cppCtx);
return null;
}
private void pruneDP(FileSinkDesc fsdesc) {
// FIXME: Support pruning dynamic partitioning.
LOG.info("DP can be rewritten to SP!");
}
}
public static NodeProcessor getFileSinkProc() {
return new ConstantPropagateFileSinkProc();
}
/**
* The Node Processor for Constant Propagation for Operators which is designed to stop propagate.
* Currently these kinds of Operators include UnionOperator and ScriptOperator.
*/
public static class ConstantPropagateStopProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
Operator<?> op = (Operator<?>) nd;
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>());
if (LOG.isDebugEnabled()) {
LOG.debug("Stop propagate constants on op " + op.getOperatorId());
}
return null;
}
}
public static NodeProcessor getStopProc() {
return new ConstantPropagateStopProc();
}
/**
* The Node Processor for Constant Propagation for ReduceSink Operators. If the RS Operator is for
* a join, then only those constants from inner join tables, or from the 'inner side' of a outer
* join (left table for left outer join and vice versa) can be propagated.
*/
public static class ConstantPropagateReduceSinkProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
ReduceSinkOperator op = (ReduceSinkOperator) nd;
ReduceSinkDesc rsDesc = op.getConf();
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, constants);
RowSchema rs = op.getSchema();
if (op.getColumnExprMap() != null && rs != null) {
for (ColumnInfo colInfo : rs.getSignature()) {
if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) {
ExprNodeDesc expr = op.getColumnExprMap().get(colInfo.getInternalName());
if (expr instanceof ExprNodeConstantDesc) {
constants.put(colInfo, expr);
}
}
}
}
if (constants.isEmpty()) {
return null;
}
if (op.getChildOperators().size() == 1
&& op.getChildOperators().get(0) instanceof JoinOperator) {
JoinOperator joinOp = (JoinOperator) op.getChildOperators().get(0);
if (skipFolding(joinOp.getConf())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip folding in outer join " + op);
}
cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>());
return null;
}
}
if (rsDesc.getDistinctColumnIndices() != null
&& !rsDesc.getDistinctColumnIndices().isEmpty()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip folding in distinct subqueries " + op);
}
cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>());
return null;
}
// key columns
ArrayList<ExprNodeDesc> newKeyEpxrs = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc desc : rsDesc.getKeyCols()) {
ExprNodeDesc newDesc = foldExpr(desc, constants, cppCtx, op, 0, false);
if (newDesc != desc && desc instanceof ExprNodeColumnDesc && newDesc instanceof ExprNodeConstantDesc) {
((ExprNodeConstantDesc)newDesc).setFoldedFromCol(((ExprNodeColumnDesc)desc).getColumn());
}
newKeyEpxrs.add(newDesc);
}
rsDesc.setKeyCols(newKeyEpxrs);
// partition columns
ArrayList<ExprNodeDesc> newPartExprs = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc desc : rsDesc.getPartitionCols()) {
ExprNodeDesc expr = foldExpr(desc, constants, cppCtx, op, 0, false);
if (expr != desc && desc instanceof ExprNodeColumnDesc
&& expr instanceof ExprNodeConstantDesc) {
((ExprNodeConstantDesc) expr).setFoldedFromCol(((ExprNodeColumnDesc) desc).getColumn());
}
newPartExprs.add(expr);
}
rsDesc.setPartitionCols(newPartExprs);
// value columns
ArrayList<ExprNodeDesc> newValExprs = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc desc : rsDesc.getValueCols()) {
newValExprs.add(foldExpr(desc, constants, cppCtx, op, 0, false));
}
rsDesc.setValueCols(newValExprs);
foldOperator(op, cppCtx);
return null;
}
/**
* Skip folding constants if there is outer join in join tree.
* @param joinDesc
* @return true if to skip.
*/
private boolean skipFolding(JoinDesc joinDesc) {
for (JoinCondDesc cond : joinDesc.getConds()) {
if (cond.getType() == JoinDesc.INNER_JOIN || cond.getType() == JoinDesc.UNIQUE_JOIN
|| cond.getType() == JoinDesc.LEFT_SEMI_JOIN) {
continue;
}
return true;
}
return false;
}
}
public static NodeProcessor getReduceSinkProc() {
return new ConstantPropagateReduceSinkProc();
}
/**
* The Node Processor for Constant Propagation for Join Operators.
*/
public static class ConstantPropagateJoinProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
JoinOperator op = (JoinOperator) nd;
JoinDesc conf = op.getConf();
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, constants);
if (constants.isEmpty()) {
return null;
}
// Note: the following code (removing folded constants in exprs) is deeply coupled with
// ColumnPruner optimizer.
// Assuming ColumnPrunner will remove constant columns so we don't deal with output columns.
// Except one case that the join operator is followed by a redistribution (RS operator).
if (op.getChildOperators().size() == 1
&& op.getChildOperators().get(0) instanceof ReduceSinkOperator) {
LOG.debug("Skip JOIN-RS structure.");
return null;
}
if (LOG.isInfoEnabled()) {
LOG.info("Old exprs " + conf.getExprs());
}
Iterator<Entry<Byte, List<ExprNodeDesc>>> itr = conf.getExprs().entrySet().iterator();
while (itr.hasNext()) {
Entry<Byte, List<ExprNodeDesc>> e = itr.next();
int tag = e.getKey();
List<ExprNodeDesc> exprs = e.getValue();
if (exprs == null) {
continue;
}
List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc expr : exprs) {
ExprNodeDesc newExpr = foldExpr(expr, constants, cppCtx, op, tag, false);
if (newExpr instanceof ExprNodeConstantDesc) {
if (LOG.isInfoEnabled()) {
LOG.info("expr " + newExpr + " fold from " + expr + " is removed.");
}
continue;
}
newExprs.add(newExpr);
}
e.setValue(newExprs);
}
if (LOG.isInfoEnabled()) {
LOG.info("New exprs " + conf.getExprs());
}
for (List<ExprNodeDesc> v : conf.getFilters().values()) {
for (int i = 0; i < v.size(); i++) {
ExprNodeDesc expr = foldExpr(v.get(i), constants, cppCtx, op, 0, false);
v.set(i, expr);
}
}
foldOperator(op, cppCtx);
return null;
}
}
public static NodeProcessor getJoinProc() {
return new ConstantPropagateJoinProc();
}
/**
* The Node Processor for Constant Propagation for Table Scan Operators.
*/
public static class ConstantPropagateTableScanProc implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs)
throws SemanticException {
TableScanOperator op = (TableScanOperator) nd;
TableScanDesc conf = op.getConf();
ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx;
Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op);
cppCtx.getOpToConstantExprs().put(op, constants);
ExprNodeGenericFuncDesc pred = conf.getFilterExpr();
if (pred == null) {
return null;
}
ExprNodeDesc constant = foldExpr(pred, constants, cppCtx, op, 0, false);
if (constant instanceof ExprNodeGenericFuncDesc) {
conf.setFilterExpr((ExprNodeGenericFuncDesc) constant);
} else {
conf.setFilterExpr(null);
}
return null;
}
}
public static NodeProcessor getTableScanProc() {
return new ConstantPropagateTableScanProc();
}
}