/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter.OpAttr;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.GenericUDAFInfo;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import com.google.common.collect.ImmutableList;
/**
* TODO:<br>
* 1. Change the output col/ExprNodeColumn names to external names.<br>
* 2. Verify if we need to use the "KEY."/"VALUE." in RS cols; switch to
* external names if possible.<br>
* 3. In ExprNode & in ColumnInfo the tableAlias/VirtualColumn is specified
* differently for different GB/RS in pipeline. Remove the different treatments.
* 4. VirtualColMap needs to be maintained
*
*/
public class HiveGBOpConvUtil {
private static enum HIVEGBPHYSICALMODE {
MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB,
MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB,
MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT,
MAP_SIDE_GB_SKEW_GBKEYS_AND_DIST_UDAF_NOT_PRESENT,
NO_MAP_SIDE_GB_NO_SKEW, NO_MAP_SIDE_GB_SKEW
};
private static class UDAFAttrs {
private boolean isDistinctUDAF;
private String udafName;
private GenericUDAFEvaluator udafEvaluator;
private final ArrayList<ExprNodeDesc> udafParams = new ArrayList<ExprNodeDesc>();
private List<Integer> udafParamsIndxInGBInfoDistExprs = new ArrayList<Integer>();
// We store the position of the argument for the function in the input.
private List<Integer> argList;
};
private static class GBInfo {
private final List<String> outputColNames = new ArrayList<String>();
private final List<String> gbKeyColNamesInInput = new ArrayList<String>();
private final List<TypeInfo> gbKeyTypes = new ArrayList<TypeInfo>();
private final List<ExprNodeDesc> gbKeys = new ArrayList<ExprNodeDesc>();
private final List<Integer> grpSets = new ArrayList<Integer>();
private boolean grpSetRqrAdditionalMRJob;
private boolean grpIdFunctionNeeded;
private final List<String> distExprNames = new ArrayList<String>();
private final List<TypeInfo> distExprTypes = new ArrayList<TypeInfo>();
private final List<ExprNodeDesc> distExprNodes = new ArrayList<ExprNodeDesc>();
private final List<List<Integer>> distColIndices = new ArrayList<List<Integer>>();
private final List<ExprNodeDesc> deDupedNonDistIrefs = new ArrayList<ExprNodeDesc>();
private final List<UDAFAttrs> udafAttrs = new ArrayList<UDAFAttrs>();
private boolean containsDistinctAggr = false;
float groupByMemoryUsage;
float memoryThreshold;
private HIVEGBPHYSICALMODE gbPhysicalPipelineMode;
};
private static HIVEGBPHYSICALMODE getAggOPMode(HiveConf hc, GBInfo gbInfo) {
HIVEGBPHYSICALMODE gbPhysicalPipelineMode = HIVEGBPHYSICALMODE.MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB;
if (hc.getBoolVar(HiveConf.ConfVars.HIVEMAPSIDEAGGREGATE)) {
if (!hc.getBoolVar(HiveConf.ConfVars.HIVEGROUPBYSKEW)) {
if (!gbInfo.grpSetRqrAdditionalMRJob) {
gbPhysicalPipelineMode = HIVEGBPHYSICALMODE.MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB;
} else {
gbPhysicalPipelineMode = HIVEGBPHYSICALMODE.MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB;
}
} else {
if (gbInfo.containsDistinctAggr || !gbInfo.gbKeys.isEmpty()) {
gbPhysicalPipelineMode = HIVEGBPHYSICALMODE.MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT;
} else {
gbPhysicalPipelineMode = HIVEGBPHYSICALMODE.MAP_SIDE_GB_SKEW_GBKEYS_AND_DIST_UDAF_NOT_PRESENT;
}
}
} else {
if (!hc.getBoolVar(HiveConf.ConfVars.HIVEGROUPBYSKEW)) {
gbPhysicalPipelineMode = HIVEGBPHYSICALMODE.NO_MAP_SIDE_GB_NO_SKEW;
} else {
gbPhysicalPipelineMode = HIVEGBPHYSICALMODE.NO_MAP_SIDE_GB_SKEW;
}
}
return gbPhysicalPipelineMode;
}
// For each of the GB op in the logical GB this should be called seperately;
// otherwise GBevaluator and expr nodes may get shared among multiple GB ops
private static GBInfo getGBInfo(HiveAggregate aggRel, OpAttr inputOpAf, HiveConf hc) throws SemanticException {
GBInfo gbInfo = new GBInfo();
// 0. Collect AggRel output col Names
gbInfo.outputColNames.addAll(aggRel.getRowType().getFieldNames());
// 1. Collect GB Keys
RelNode aggInputRel = aggRel.getInput();
ExprNodeConverter exprConv = new ExprNodeConverter(inputOpAf.tabAlias,
aggInputRel.getRowType(), new HashSet<Integer>(), aggRel.getCluster().getTypeFactory(),
true);
ExprNodeDesc tmpExprNodeDesc;
for (int i : aggRel.getGroupSet()) {
RexInputRef iRef = new RexInputRef(i, aggInputRel.getRowType().getFieldList()
.get(i).getType());
tmpExprNodeDesc = iRef.accept(exprConv);
gbInfo.gbKeys.add(tmpExprNodeDesc);
gbInfo.gbKeyColNamesInInput.add(aggInputRel.getRowType().getFieldNames().get(i));
gbInfo.gbKeyTypes.add(tmpExprNodeDesc.getTypeInfo());
}
// 2. Collect Grouping Set info
if (aggRel.indicator) {
// 2.1 Translate Grouping set col bitset
ImmutableList<ImmutableBitSet> lstGrpSet = aggRel.getGroupSets();
int bitmap = 0;
for (ImmutableBitSet grpSet : lstGrpSet) {
bitmap = 0;
for (Integer bitIdx : grpSet.asList()) {
bitmap = SemanticAnalyzer.setBit(bitmap, bitIdx);
}
gbInfo.grpSets.add(bitmap);
}
Collections.sort(gbInfo.grpSets);
// 2.2 Check if GRpSet require additional MR Job
gbInfo.grpSetRqrAdditionalMRJob = gbInfo.grpSets.size() > hc
.getIntVar(HiveConf.ConfVars.HIVE_NEW_JOB_GROUPING_SET_CARDINALITY);
// 2.3 Check if GROUPING_ID needs to be projected out
if (!aggRel.getAggCallList().isEmpty()
&& (aggRel.getAggCallList().get(aggRel.getAggCallList().size() - 1).getAggregation() == HiveGroupingID.INSTANCE)) {
gbInfo.grpIdFunctionNeeded = true;
}
}
// 3. Walk through UDAF & Collect Distinct Info
Set<Integer> distinctRefs = new HashSet<Integer>();
Map<Integer, Integer> distParamInRefsToOutputPos = new HashMap<Integer, Integer>();
for (AggregateCall aggCall : aggRel.getAggCallList()) {
if ((aggCall.getAggregation() == HiveGroupingID.INSTANCE) || !aggCall.isDistinct()) {
continue;
}
List<Integer> argLst = new ArrayList<Integer>(aggCall.getArgList());
List<String> argNames = HiveCalciteUtil.getFieldNames(argLst, aggInputRel);
ExprNodeDesc distinctExpr;
for (int i = 0; i < argLst.size(); i++) {
if (!distinctRefs.contains(argLst.get(i))) {
distinctRefs.add(argLst.get(i));
distinctExpr = HiveCalciteUtil.getExprNode(argLst.get(i), aggInputRel, exprConv);
// Only distinct nodes that are NOT part of the key should be added to distExprNodes
if (ExprNodeDescUtils.indexOf(distinctExpr, gbInfo.gbKeys) < 0) {
distParamInRefsToOutputPos.put(argLst.get(i), gbInfo.distExprNodes.size());
gbInfo.distExprNodes.add(distinctExpr);
gbInfo.distExprNames.add(argNames.get(i));
gbInfo.distExprTypes.add(distinctExpr.getTypeInfo());
}
}
}
}
// 4. Walk through UDAF & Collect UDAF Info
Set<Integer> deDupedNonDistIrefsSet = new HashSet<Integer>();
for (AggregateCall aggCall : aggRel.getAggCallList()) {
if (aggCall.getAggregation() == HiveGroupingID.INSTANCE) {
continue;
}
UDAFAttrs udafAttrs = new UDAFAttrs();
List<ExprNodeDesc> argExps = HiveCalciteUtil.getExprNodes(aggCall.getArgList(), aggInputRel,
inputOpAf.tabAlias);
udafAttrs.udafParams.addAll(argExps);
udafAttrs.udafName = aggCall.getAggregation().getName();
udafAttrs.argList = aggCall.getArgList();
udafAttrs.isDistinctUDAF = aggCall.isDistinct();
List<Integer> argLst = new ArrayList<Integer>(aggCall.getArgList());
List<Integer> distColIndicesOfUDAF = new ArrayList<Integer>();
List<Integer> distUDAFParamsIndxInDistExprs = new ArrayList<Integer>();
for (int i = 0; i < argLst.size(); i++) {
// NOTE: distinct expr can be part of of GB key
if (udafAttrs.isDistinctUDAF) {
ExprNodeDesc argExpr = argExps.get(i);
Integer found = ExprNodeDescUtils.indexOf(argExpr, gbInfo.gbKeys);
distColIndicesOfUDAF.add(found < 0 ? distParamInRefsToOutputPos.get(argLst.get(i)) + gbInfo.gbKeys.size() +
(gbInfo.grpSets.size() > 0 ? 1 : 0) : found);
distUDAFParamsIndxInDistExprs.add(distParamInRefsToOutputPos.get(argLst.get(i)));
} else {
// TODO: this seems wrong (following what Hive Regular does)
if (!distParamInRefsToOutputPos.containsKey(argLst.get(i))
&& !deDupedNonDistIrefsSet.contains(argLst.get(i))) {
deDupedNonDistIrefsSet.add(argLst.get(i));
gbInfo.deDupedNonDistIrefs.add(udafAttrs.udafParams.get(i));
}
}
}
if (udafAttrs.isDistinctUDAF) {
gbInfo.containsDistinctAggr = true;
udafAttrs.udafParamsIndxInGBInfoDistExprs = distUDAFParamsIndxInDistExprs;
gbInfo.distColIndices.add(distColIndicesOfUDAF);
}
// special handling for count, similar to PlanModifierForASTConv::replaceEmptyGroupAggr()
udafAttrs.udafEvaluator = SemanticAnalyzer.getGenericUDAFEvaluator(udafAttrs.udafName,
new ArrayList<ExprNodeDesc>(udafAttrs.udafParams), new ASTNode(),
udafAttrs.isDistinctUDAF, udafAttrs.udafParams.size() == 0 &&
"count".equalsIgnoreCase(udafAttrs.udafName) ? true : false);
gbInfo.udafAttrs.add(udafAttrs);
}
// 4. Gather GB Memory threshold
gbInfo.groupByMemoryUsage = HiveConf.getFloatVar(hc, HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
gbInfo.memoryThreshold = HiveConf.getFloatVar(hc, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
// 5. Gather GB Physical pipeline (based on user config & Grping Sets size)
gbInfo.gbPhysicalPipelineMode = getAggOPMode(hc, gbInfo);
return gbInfo;
}
static OpAttr translateGB(OpAttr inputOpAf, HiveAggregate aggRel, HiveConf hc)
throws SemanticException {
OpAttr translatedGBOpAttr = null;
GBInfo gbInfo = getGBInfo(aggRel, inputOpAf, hc);
switch (gbInfo.gbPhysicalPipelineMode) {
case MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB:
translatedGBOpAttr = genMapSideGBNoSkewNoAddMRJob(inputOpAf, aggRel, gbInfo);
break;
case MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB:
translatedGBOpAttr = genMapSideGBNoSkewAddMRJob(inputOpAf, aggRel, gbInfo);
break;
case MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT:
translatedGBOpAttr = genMapSideGBSkewGBKeysOrDistUDAFPresent(inputOpAf, aggRel, gbInfo);
break;
case MAP_SIDE_GB_SKEW_GBKEYS_AND_DIST_UDAF_NOT_PRESENT:
translatedGBOpAttr = genMapSideGBSkewGBKeysAndDistUDAFNotPresent(inputOpAf, aggRel, gbInfo);
break;
case NO_MAP_SIDE_GB_NO_SKEW:
translatedGBOpAttr = genNoMapSideGBNoSkew(inputOpAf, aggRel, gbInfo);
break;
case NO_MAP_SIDE_GB_SKEW:
translatedGBOpAttr = genNoMapSideGBSkew(inputOpAf, aggRel, gbInfo);
break;
}
return translatedGBOpAttr;
}
/**
* GB-RS-GB1
*
* Construct GB-RS-GB Pipe line. User has enabled Map Side GB, specified no
* skew and Grp Set is below the threshold.
*
* @param inputOpAf
* @param aggRel
* @param gbInfo
* @return
* @throws SemanticException
*/
private static OpAttr genMapSideGBNoSkewNoAddMRJob(OpAttr inputOpAf, HiveAggregate aggRel,
GBInfo gbInfo) throws SemanticException {
OpAttr mapSideGB = null;
OpAttr mapSideRS = null;
OpAttr reduceSideGB = null;
// 1. Insert MapSide GB
mapSideGB = genMapSideGB(inputOpAf, gbInfo);
// 2. Insert MapSide RS
mapSideRS = genMapSideGBRS(mapSideGB, gbInfo);
// 3. Insert ReduceSide GB
reduceSideGB = genReduceSideGB1(mapSideRS, gbInfo, false, false, GroupByDesc.Mode.MERGEPARTIAL);
return reduceSideGB;
}
/**
* GB-RS-GB1-RS-GB2
*/
private static OpAttr genGBRSGBRSGBOpPipeLine(OpAttr inputOpAf, HiveAggregate aggRel,
GBInfo gbInfo) throws SemanticException {
OpAttr mapSideGB = null;
OpAttr mapSideRS = null;
OpAttr reduceSideGB1 = null;
OpAttr reduceSideRS = null;
OpAttr reduceSideGB2 = null;
// 1. Insert MapSide GB
mapSideGB = genMapSideGB(inputOpAf, gbInfo);
// 2. Insert MapSide RS
mapSideRS = genMapSideGBRS(mapSideGB, gbInfo);
// 3. Insert ReduceSide GB1
boolean computeGrpSet = (gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT) ? false : true;
reduceSideGB1 = genReduceSideGB1(mapSideRS, gbInfo, computeGrpSet, false, GroupByDesc.Mode.PARTIALS);
// 4. Insert RS on reduce side with Reduce side GB as input
reduceSideRS = genReduceGBRS(reduceSideGB1, gbInfo);
// 5. Insert ReduceSide GB2
reduceSideGB2 = genReduceSideGB2(reduceSideRS, gbInfo);
return reduceSideGB2;
}
/**
* GB-RS-GB1-RS-GB2
*
* @param inputOpAf
* @param aggRel
* @param gbInfo
* @return
* @throws SemanticException
*/
private static OpAttr genMapSideGBNoSkewAddMRJob(OpAttr inputOpAf, HiveAggregate aggRel,
GBInfo gbInfo) throws SemanticException {
// 1. Sanity check
if (gbInfo.containsDistinctAggr) {
String errorMsg = "The number of rows per input row due to grouping sets is "
+ gbInfo.grpSets.size();
throw new SemanticException(
ErrorMsg.HIVE_GROUPING_SETS_THRESHOLD_NOT_ALLOWED_WITH_DISTINCTS.getMsg(errorMsg));
}
// 2. Gen GB-RS-GB-RS-GB pipeline
return genGBRSGBRSGBOpPipeLine(inputOpAf, aggRel, gbInfo);
}
/**
* GB-RS-GB1-RS-GB2
*
* @param inputOpAf
* @param aggRel
* @param gbInfo
* @return
* @throws SemanticException
*/
private static OpAttr genMapSideGBSkewGBKeysOrDistUDAFPresent(OpAttr inputOpAf,
HiveAggregate aggRel, GBInfo gbInfo) throws SemanticException {
// 1. Sanity check
if (gbInfo.grpSetRqrAdditionalMRJob) {
String errorMsg = "The number of rows per input row due to grouping sets is "
+ gbInfo.grpSets.size();
throw new SemanticException(
ErrorMsg.HIVE_GROUPING_SETS_THRESHOLD_NOT_ALLOWED_WITH_SKEW.getMsg(errorMsg));
}
// 2. Gen GB-RS-GB-RS-GB pipeline
return genGBRSGBRSGBOpPipeLine(inputOpAf, aggRel, gbInfo);
}
/**
* GB-RS-GB2
*
* @param inputOpAf
* @param aggRel
* @param gbInfo
* @return
* @throws SemanticException
*/
private static OpAttr genMapSideGBSkewGBKeysAndDistUDAFNotPresent(OpAttr inputOpAf,
HiveAggregate aggRel, GBInfo gbInfo) throws SemanticException {
OpAttr mapSideGB = null;
OpAttr mapSideRS = null;
OpAttr reduceSideGB2 = null;
// 1. Sanity check
if (gbInfo.grpSetRqrAdditionalMRJob) {
String errorMsg = "The number of rows per input row due to grouping sets is "
+ gbInfo.grpSets.size();
throw new SemanticException(
ErrorMsg.HIVE_GROUPING_SETS_THRESHOLD_NOT_ALLOWED_WITH_SKEW.getMsg(errorMsg));
}
// 1. Insert MapSide GB
mapSideGB = genMapSideGB(inputOpAf, gbInfo);
// 2. Insert MapSide RS
mapSideRS = genMapSideGBRS(mapSideGB, gbInfo);
// 3. Insert ReduceSide GB2
reduceSideGB2 = genReduceSideGB2(mapSideRS, gbInfo);
return reduceSideGB2;
}
/**
* RS-Gb1
*
* @param inputOpAf
* @param aggRel
* @param gbInfo
* @return
* @throws SemanticException
*/
private static OpAttr genNoMapSideGBNoSkew(OpAttr inputOpAf, HiveAggregate aggRel, GBInfo gbInfo)
throws SemanticException {
OpAttr mapSideRS = null;
OpAttr reduceSideGB1NoMapGB = null;
// 1. Insert MapSide RS
mapSideRS = genMapSideRS(inputOpAf, gbInfo);
// 2. Insert ReduceSide GB
reduceSideGB1NoMapGB = genReduceSideGB1NoMapGB(mapSideRS, gbInfo, GroupByDesc.Mode.COMPLETE);
return reduceSideGB1NoMapGB;
}
/**
* RS-GB1-RS-GB2
*
* @param inputOpAf
* @param aggRel
* @param gbInfo
* @return
* @throws SemanticException
*/
private static OpAttr genNoMapSideGBSkew(OpAttr inputOpAf, HiveAggregate aggRel, GBInfo gbInfo)
throws SemanticException {
OpAttr mapSideRS = null;
OpAttr reduceSideGB1NoMapGB = null;
OpAttr reduceSideRS = null;
OpAttr reduceSideGB2 = null;
// 1. Insert MapSide RS
mapSideRS = genMapSideRS(inputOpAf, gbInfo);
// 2. Insert ReduceSide GB
reduceSideGB1NoMapGB = genReduceSideGB1NoMapGB(mapSideRS, gbInfo, GroupByDesc.Mode.PARTIAL1);
// 3. Insert RS on reduce side with Reduce side GB as input
reduceSideRS = genReduceGBRS(reduceSideGB1NoMapGB, gbInfo);
// 4. Insert ReduceSide GB2
reduceSideGB2 = genReduceSideGB2(reduceSideRS, gbInfo);
return reduceSideGB2;
}
private static int getParallelismForReduceSideRS(GBInfo gbInfo) {
int degreeOfParallelism = 0;
switch (gbInfo.gbPhysicalPipelineMode) {
case MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB:
case MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT:
case NO_MAP_SIDE_GB_SKEW:
if (gbInfo.gbKeys.isEmpty()) {
degreeOfParallelism = 1;
} else {
degreeOfParallelism = -1;
}
break;
default:
throw new RuntimeException(
"Unable to determine Reducer Parallelism - Invalid Physical Mode: "
+ gbInfo.gbPhysicalPipelineMode);
}
return degreeOfParallelism;
}
private static int getParallelismForMapSideRS(GBInfo gbInfo) {
int degreeOfParallelism = 0;
switch (gbInfo.gbPhysicalPipelineMode) {
case MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB:
case MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB:
case NO_MAP_SIDE_GB_NO_SKEW:
if (gbInfo.gbKeys.isEmpty()) {
degreeOfParallelism = 1;
} else {
degreeOfParallelism = -1;
}
break;
case NO_MAP_SIDE_GB_SKEW:
case MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT:
degreeOfParallelism = -1;
break;
case MAP_SIDE_GB_SKEW_GBKEYS_AND_DIST_UDAF_NOT_PRESENT:
degreeOfParallelism = 1;
break;
default:
throw new RuntimeException(
"Unable to determine Reducer Parallelism - Invalid Physical Mode: "
+ gbInfo.gbPhysicalPipelineMode);
}
return degreeOfParallelism;
}
private static int getNumPartFieldsForReduceSideRS(GBInfo gbInfo) {
int numPartFields = 0;
switch (gbInfo.gbPhysicalPipelineMode) {
case MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB:
numPartFields = gbInfo.gbKeys.size() + 1;
break;
case MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT:
case NO_MAP_SIDE_GB_SKEW:
numPartFields = gbInfo.gbKeys.size();
break;
default:
throw new RuntimeException(
"Unable to determine Number of Partition Fields - Invalid Physical Mode: "
+ gbInfo.gbPhysicalPipelineMode);
}
return numPartFields;
}
private static int getNumPartFieldsForMapSideRS(GBInfo gbInfo) {
int numPartFields = 0;
switch (gbInfo.gbPhysicalPipelineMode) {
case MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB:
case MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB:
case MAP_SIDE_GB_SKEW_GBKEYS_AND_DIST_UDAF_NOT_PRESENT:
case NO_MAP_SIDE_GB_NO_SKEW:
numPartFields += gbInfo.gbKeys.size();
break;
case NO_MAP_SIDE_GB_SKEW:
case MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT:
if (gbInfo.containsDistinctAggr) {
numPartFields = Integer.MAX_VALUE;
} else {
numPartFields = -1;
}
break;
default:
throw new RuntimeException(
"Unable to determine Number of Partition Fields - Invalid Physical Mode: "
+ gbInfo.gbPhysicalPipelineMode);
}
return numPartFields;
}
private static boolean inclGrpSetInReduceSide(GBInfo gbInfo) {
boolean inclGrpSet = false;
if (gbInfo.grpSets.size() > 0
&& (gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.MAP_SIDE_GB_NO_SKEW_ADD_MR_JOB || gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT)) {
inclGrpSet = true;
}
return inclGrpSet;
}
private static boolean inclGrpSetInMapSide(GBInfo gbInfo) {
boolean inclGrpSet = false;
if (gbInfo.grpSets.size() > 0
&& ((gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB) ||
gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT)) {
inclGrpSet = true;
}
return inclGrpSet;
}
private static OpAttr genReduceGBRS(OpAttr inputOpAf, GBInfo gbInfo) throws SemanticException {
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
ArrayList<String> outputColumnNames = new ArrayList<String>();
ArrayList<ColumnInfo> colInfoLst = new ArrayList<ColumnInfo>();
GroupByOperator reduceSideGB1 = (GroupByOperator) inputOpAf.inputs.get(0);
List<ColumnInfo> gb1ColInfoLst = reduceSideGB1.getSchema().getSignature();
ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForRS(reduceSideGB1, 0,
gbInfo.gbKeys.size() - 1, outputColumnNames, false, colInfoLst, colExprMap, true, true);
if (inclGrpSetInReduceSide(gbInfo)) {
addGrpSetCol(false, gb1ColInfoLst.get(reduceKeys.size()).getInternalName(), true, reduceKeys,
outputColumnNames, colInfoLst, colExprMap);
}
ArrayList<ExprNodeDesc> reduceValues = getValueKeysForRS(reduceSideGB1, reduceSideGB1.getConf()
.getKeys().size(), outputColumnNames, colInfoLst, colExprMap, true, true);
ReduceSinkOperator rsOp = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(PlanUtils
.getReduceSinkDesc(reduceKeys, reduceValues, outputColumnNames, true, -1,
getNumPartFieldsForReduceSideRS(gbInfo), getParallelismForReduceSideRS(gbInfo),
AcidUtils.Operation.NOT_ACID), new RowSchema(colInfoLst), reduceSideGB1);
rsOp.setColumnExprMap(colExprMap);
return new OpAttr("", new HashSet<Integer>(), rsOp);
}
private static OpAttr genMapSideGBRS(OpAttr inputOpAf, GBInfo gbInfo) throws SemanticException {
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
List<String> outputKeyColumnNames = new ArrayList<String>();
List<String> outputValueColumnNames = new ArrayList<String>();
ArrayList<ColumnInfo> colInfoLst = new ArrayList<ColumnInfo>();
GroupByOperator mapGB = (GroupByOperator) inputOpAf.inputs.get(0);
ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForRS(mapGB, 0, gbInfo.gbKeys.size() - 1,
outputKeyColumnNames, false, colInfoLst, colExprMap, false, false);
int keyLength = reduceKeys.size();
if (inclGrpSetInMapSide(gbInfo)) {
addGrpSetCol(false, SemanticAnalyzer.getColumnInternalName(reduceKeys.size()), true,
reduceKeys, outputKeyColumnNames, colInfoLst, colExprMap);
keyLength++;
}
if (mapGB.getConf().getKeys().size() > reduceKeys.size()) {
// NOTE: All dist cols have single output col name;
reduceKeys.addAll(getReduceKeysForRS(mapGB, reduceKeys.size(), mapGB.getConf().getKeys()
.size() - 1, outputKeyColumnNames, true, colInfoLst, colExprMap, false, false));
} else if (!gbInfo.distColIndices.isEmpty()) {
// This is the case where distinct cols are part of GB Keys in which case
// we still need to add it to out put col names
outputKeyColumnNames.add(SemanticAnalyzer.getColumnInternalName(reduceKeys.size()));
}
ArrayList<ExprNodeDesc> reduceValues = getValueKeysForRS(mapGB, mapGB.getConf().getKeys()
.size(), outputValueColumnNames, colInfoLst, colExprMap, false, false);
ReduceSinkOperator rsOp = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(PlanUtils
.getReduceSinkDesc(reduceKeys, keyLength, reduceValues, gbInfo.distColIndices,
outputKeyColumnNames, outputValueColumnNames, true, -1, getNumPartFieldsForMapSideRS(
gbInfo), getParallelismForMapSideRS(gbInfo), AcidUtils.Operation.NOT_ACID),
new RowSchema(colInfoLst), mapGB);
rsOp.setColumnExprMap(colExprMap);
return new OpAttr("", new HashSet<Integer>(), rsOp);
}
private static OpAttr genMapSideRS(OpAttr inputOpAf, GBInfo gbInfo) throws SemanticException {
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
List<String> outputKeyColumnNames = new ArrayList<String>();
List<String> outputValueColumnNames = new ArrayList<String>();
ArrayList<ColumnInfo> colInfoLst = new ArrayList<ColumnInfo>();
String outputColName;
// 1. Add GB Keys to reduce keys
ArrayList<ExprNodeDesc> reduceKeys= new ArrayList<ExprNodeDesc>();
for (int i = 0; i < gbInfo.gbKeys.size(); i++) {
//gbInfo already has ExprNode for gbkeys
reduceKeys.add(gbInfo.gbKeys.get(i));
String colOutputName = SemanticAnalyzer.getColumnInternalName(i);
outputKeyColumnNames.add(colOutputName);
colInfoLst.add(new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + colOutputName, gbInfo.gbKeyTypes.get(i), "", false));
colExprMap.put(colOutputName, gbInfo.gbKeys.get(i));
}
// Note: GROUPING SETS are not allowed with map side aggregation set to false so we don't have to worry about it
int keyLength = reduceKeys.size();
// 2. Add Dist UDAF args to reduce keys
if (gbInfo.containsDistinctAggr) {
// TODO: Why is this needed (doesn't represent any cols)
String udafName = SemanticAnalyzer.getColumnInternalName(reduceKeys.size());
outputKeyColumnNames.add(udafName);
for (int i = 0; i < gbInfo.distExprNodes.size(); i++) {
reduceKeys.add(gbInfo.distExprNodes.get(i));
//this part of reduceKeys is later used to create column names strictly for non-distinct aggregates
// with parameters same as distinct keys which expects _col0 at the end. So we always append
// _col0 at the end instead of _col<i>
outputColName = SemanticAnalyzer.getColumnInternalName(0);
String field = Utilities.ReduceField.KEY.toString() + "." + udafName + ":" + i + "."
+ outputColName;
ColumnInfo colInfo = new ColumnInfo(field, gbInfo.distExprNodes.get(i).getTypeInfo(), null,
false);
colInfoLst.add(colInfo);
colExprMap.put(field, gbInfo.distExprNodes.get(i));
}
}
// 3. Add UDAF args deduped to reduce values
ArrayList<ExprNodeDesc> reduceValues = new ArrayList<ExprNodeDesc>();
for (int i = 0; i < gbInfo.deDupedNonDistIrefs.size(); i++) {
reduceValues.add(gbInfo.deDupedNonDistIrefs.get(i));
outputColName = SemanticAnalyzer.getColumnInternalName(reduceValues.size() - 1);
outputValueColumnNames.add(outputColName);
String field = Utilities.ReduceField.VALUE.toString() + "." + outputColName;
colInfoLst.add(new ColumnInfo(field, reduceValues.get(reduceValues.size() - 1).getTypeInfo(),
null, false));
colExprMap.put(field, reduceValues.get(reduceValues.size() - 1));
}
// 4. Gen RS
ReduceSinkOperator rsOp = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(PlanUtils
.getReduceSinkDesc(reduceKeys, keyLength, reduceValues,
gbInfo.distColIndices, outputKeyColumnNames,
outputValueColumnNames, true, -1, getNumPartFieldsForMapSideRS(gbInfo),
getParallelismForMapSideRS(gbInfo), AcidUtils.Operation.NOT_ACID), new RowSchema(
colInfoLst), inputOpAf.inputs.get(0));
rsOp.setColumnExprMap(colExprMap);
return new OpAttr("", new HashSet<Integer>(), rsOp);
}
private static OpAttr genReduceSideGB2(OpAttr inputOpAf, GBInfo gbInfo) throws SemanticException {
ArrayList<String> outputColNames = new ArrayList<String>();
ArrayList<ColumnInfo> colInfoLst = new ArrayList<ColumnInfo>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
String colOutputName = null;
ReduceSinkOperator rs = (ReduceSinkOperator) inputOpAf.inputs.get(0);
List<ColumnInfo> rsColInfoLst = rs.getSchema().getSignature();
ColumnInfo ci;
// 1. Build GB Keys, grouping set starting position
// 1.1 First Add original GB Keys
ArrayList<ExprNodeDesc> gbKeys = ExprNodeDescUtils.genExprNodeDesc(rs, 0,
gbInfo.gbKeys.size() - 1, false, false);
for (int i = 0; i < gbInfo.gbKeys.size(); i++) {
ci = rsColInfoLst.get(i);
colOutputName = gbInfo.outputColNames.get(i);
outputColNames.add(colOutputName);
colInfoLst.add(new ColumnInfo(colOutputName, ci.getType(), "", false));
colExprMap.put(colOutputName, gbKeys.get(i));
}
// 1.2 Add GrpSet Col
int groupingSetsPosition = -1;
if (inclGrpSetInReduceSide(gbInfo) && gbInfo.grpIdFunctionNeeded) {
groupingSetsPosition = gbKeys.size();
ExprNodeDesc grpSetColExpr = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
rsColInfoLst.get(groupingSetsPosition).getInternalName(), null, false);
gbKeys.add(grpSetColExpr);
colOutputName = gbInfo.outputColNames.get(gbInfo.outputColNames.size() - 1);
;
outputColNames.add(colOutputName);
colInfoLst.add(new ColumnInfo(colOutputName, TypeInfoFactory.stringTypeInfo, null, true));
colExprMap.put(colOutputName, grpSetColExpr);
}
// 2. Add UDAF
UDAFAttrs udafAttr;
ArrayList<AggregationDesc> aggregations = new ArrayList<AggregationDesc>();
int udafStartPosInGBInfOutputColNames = gbInfo.grpSets.isEmpty() ? gbInfo.gbKeys.size()
: gbInfo.gbKeys.size() * 2;
int udafStartPosInInputRS = gbInfo.grpSets.isEmpty() ? gbInfo.gbKeys.size() : gbInfo.gbKeys.size() + 1;
for (int i = 0; i < gbInfo.udafAttrs.size(); i++) {
udafAttr = gbInfo.udafAttrs.get(i);
ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
aggParameters.add(new ExprNodeColumnDesc(rsColInfoLst.get(udafStartPosInInputRS + i)));
colOutputName = gbInfo.outputColNames.get(udafStartPosInGBInfOutputColNames + i);
outputColNames.add(colOutputName);
Mode udafMode = SemanticAnalyzer.groupByDescModeToUDAFMode(GroupByDesc.Mode.FINAL,
udafAttr.isDistinctUDAF);
GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(udafAttr.udafEvaluator, udafMode,
aggParameters);
aggregations.add(new AggregationDesc(udafAttr.udafName.toLowerCase(),
udaf.genericUDAFEvaluator, udaf.convertedParameters, false, udafMode));
colInfoLst.add(new ColumnInfo(colOutputName, udaf.returnType, "", false));
}
Operator rsGBOp2 = OperatorFactory.getAndMakeChild(new GroupByDesc(GroupByDesc.Mode.FINAL,
outputColNames, gbKeys, aggregations, false, gbInfo.groupByMemoryUsage,
gbInfo.memoryThreshold, null, false, groupingSetsPosition, gbInfo.containsDistinctAggr),
new RowSchema(colInfoLst), rs);
rsGBOp2.setColumnExprMap(colExprMap);
// TODO: Shouldn't we propgate vc? is it vc col from tab or all vc
return new OpAttr("", new HashSet<Integer>(), rsGBOp2);
}
private static OpAttr genReduceSideGB1(OpAttr inputOpAf, GBInfo gbInfo, boolean computeGrpSet,
boolean propagateConstInDistinctUDAF, GroupByDesc.Mode gbMode) throws SemanticException {
ArrayList<String> outputColNames = new ArrayList<String>();
ArrayList<ColumnInfo> colInfoLst = new ArrayList<ColumnInfo>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
String colOutputName = null;
ReduceSinkOperator rs = (ReduceSinkOperator) inputOpAf.inputs.get(0);
List<ColumnInfo> rsColInfoLst = rs.getSchema().getSignature();
ColumnInfo ci;
boolean finalGB = (gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.MAP_SIDE_GB_NO_SKEW_NO_ADD_MR_JOB);
// 1. Build GB Keys, grouping set starting position
// 1.1 First Add original GB Keys
ArrayList<ExprNodeDesc> gbKeys = ExprNodeDescUtils.genExprNodeDesc(rs, 0,
gbInfo.gbKeys.size() - 1, false, false);
for (int i = 0; i < gbInfo.gbKeys.size(); i++) {
ci = rsColInfoLst.get(i);
if (finalGB) {
colOutputName = gbInfo.outputColNames.get(i);
} else {
colOutputName = SemanticAnalyzer.getColumnInternalName(i);
}
outputColNames.add(colOutputName);
colInfoLst.add(new ColumnInfo(colOutputName, ci.getType(), "", false));
colExprMap.put(colOutputName, gbKeys.get(i));
}
// 1.2 Add GrpSet Col
int groupingSetsColPosition = -1;
if ((!finalGB && gbInfo.grpSets.size() > 0) || (finalGB && gbInfo.grpIdFunctionNeeded)) {
groupingSetsColPosition = gbInfo.gbKeys.size();
if (computeGrpSet) {
// GrpSet Col needs to be constructed
gbKeys.add(new ExprNodeConstantDesc("0"));
} else {
// GrpSet Col already part of input RS
// TODO: Can't we just copy the ExprNodeDEsc from input (Do we need to
// explicitly set table alias to null & VC to false
gbKeys.addAll(ExprNodeDescUtils.genExprNodeDesc(rs, groupingSetsColPosition,
groupingSetsColPosition, false, true));
}
colOutputName = SemanticAnalyzer.getColumnInternalName(groupingSetsColPosition);
if (finalGB) {
colOutputName = gbInfo.outputColNames.get(gbInfo.outputColNames.size() - 1);
}
outputColNames.add(colOutputName);
colInfoLst.add(new ColumnInfo(colOutputName, TypeInfoFactory.stringTypeInfo, null, true));
colExprMap.put(colOutputName, gbKeys.get(groupingSetsColPosition));
}
// 2. Walk through UDAF and add them to GB
String lastReduceKeyColName = null;
if (!rs.getConf().getOutputKeyColumnNames().isEmpty()) {
lastReduceKeyColName = rs.getConf().getOutputKeyColumnNames()
.get(rs.getConf().getOutputKeyColumnNames().size() - 1);
}
int numDistinctUDFs = 0;
int distinctStartPosInReduceKeys = gbKeys.size();
List<ExprNodeDesc> reduceValues = rs.getConf().getValueCols();
ArrayList<AggregationDesc> aggregations = new ArrayList<AggregationDesc>();
int udafColStartPosInOriginalGB = (gbInfo.grpSets.size() > 0) ? gbInfo.gbKeys.size() * 2
: gbInfo.gbKeys.size();
int udafColStartPosInRS = rs.getConf().getKeyCols().size();
for (int i = 0; i < gbInfo.udafAttrs.size(); i++) {
UDAFAttrs udafAttr = gbInfo.udafAttrs.get(i);
ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
if (udafAttr.isDistinctUDAF) {
ColumnInfo rsDistUDAFParamColInfo;
ExprNodeDesc distinctUDAFParam;
ExprNodeDesc constantPropDistinctUDAFParam;
for (int j = 0; j < udafAttr.udafParamsIndxInGBInfoDistExprs.size(); j++) {
rsDistUDAFParamColInfo = rsColInfoLst.get(distinctStartPosInReduceKeys + j);
String rsDistUDAFParamName = rsDistUDAFParamColInfo.getInternalName();
// TODO: verify if this is needed
if (lastReduceKeyColName != null) {
rsDistUDAFParamName = Utilities.ReduceField.KEY.name() + "." + lastReduceKeyColName
+ ":" + numDistinctUDFs + "." + SemanticAnalyzer.getColumnInternalName(j);
}
distinctUDAFParam = new ExprNodeColumnDesc(rsDistUDAFParamColInfo.getType(),
rsDistUDAFParamName, rsDistUDAFParamColInfo.getTabAlias(),
rsDistUDAFParamColInfo.getIsVirtualCol());
if (propagateConstInDistinctUDAF) {
// TODO: Implement propConstDistUDAFParams
constantPropDistinctUDAFParam = SemanticAnalyzer
.isConstantParameterInAggregationParameters(
rsDistUDAFParamColInfo.getInternalName(), reduceValues);
if (constantPropDistinctUDAFParam != null) {
distinctUDAFParam = constantPropDistinctUDAFParam;
}
}
aggParameters.add(distinctUDAFParam);
}
numDistinctUDFs++;
} else {
aggParameters.add(new ExprNodeColumnDesc(rsColInfoLst.get(udafColStartPosInRS + i)));
}
Mode udafMode = SemanticAnalyzer.groupByDescModeToUDAFMode(gbMode, udafAttr.isDistinctUDAF);
GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(udafAttr.udafEvaluator, udafMode,
aggParameters);
aggregations.add(new AggregationDesc(udafAttr.udafName.toLowerCase(),
udaf.genericUDAFEvaluator, udaf.convertedParameters,
(gbMode != GroupByDesc.Mode.FINAL && udafAttr.isDistinctUDAF), udafMode));
if (finalGB) {
colOutputName = gbInfo.outputColNames.get(udafColStartPosInOriginalGB + i);
} else {
colOutputName = SemanticAnalyzer.getColumnInternalName(gbKeys.size() + aggregations.size()
- 1);
}
colInfoLst.add(new ColumnInfo(colOutputName, udaf.returnType, "", false));
outputColNames.add(colOutputName);
}
// Nothing special needs to be done for grouping sets if
// this is the final group by operator, and multiple rows corresponding to
// the
// grouping sets have been generated upstream.
// However, if an addition MR job has been created to handle grouping sets,
// additional rows corresponding to grouping sets need to be created here.
//TODO: Clean up/refactor assumptions
boolean includeGrpSetInGBDesc = (gbInfo.grpSets.size() > 0)
&& !finalGB
&& !(gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.MAP_SIDE_GB_SKEW_GBKEYS_OR_DIST_UDAF_PRESENT);
Operator rsGBOp = OperatorFactory.getAndMakeChild(new GroupByDesc(gbMode, outputColNames,
gbKeys, aggregations, gbInfo.groupByMemoryUsage, gbInfo.memoryThreshold, gbInfo.grpSets,
includeGrpSetInGBDesc, groupingSetsColPosition, gbInfo.containsDistinctAggr),
new RowSchema(colInfoLst), rs);
rsGBOp.setColumnExprMap(colExprMap);
return new OpAttr("", new HashSet<Integer>(), rsGBOp);
}
/**
* RS-GB0
*
* @param inputOpAf
* @param gbInfo
* @param gbMode
* @return
* @throws SemanticException
*/
private static OpAttr genReduceSideGB1NoMapGB(OpAttr inputOpAf, GBInfo gbInfo,
GroupByDesc.Mode gbMode) throws SemanticException {
ArrayList<String> outputColNames = new ArrayList<String>();
ArrayList<ColumnInfo> colInfoLst = new ArrayList<ColumnInfo>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
String colOutputName = null;
ReduceSinkOperator rs = (ReduceSinkOperator) inputOpAf.inputs.get(0);
List<ColumnInfo> rsColInfoLst = rs.getSchema().getSignature();
ColumnInfo ci;
boolean useOriginalGBNames = (gbInfo.gbPhysicalPipelineMode == HIVEGBPHYSICALMODE.NO_MAP_SIDE_GB_NO_SKEW);
// 1. Build GB Keys, grouping set starting position
// 1.1 First Add original GB Keys
ArrayList<ExprNodeDesc> gbKeys = ExprNodeDescUtils.genExprNodeDesc(rs, 0,
gbInfo.gbKeys.size() - 1, true, false);
for (int i = 0; i < gbInfo.gbKeys.size(); i++) {
ci = rsColInfoLst.get(i);
if (useOriginalGBNames) {
colOutputName = gbInfo.outputColNames.get(i);
} else {
colOutputName = SemanticAnalyzer.getColumnInternalName(i);
}
outputColNames.add(colOutputName);
colInfoLst.add(new ColumnInfo(colOutputName, ci.getType(), null, false));
colExprMap.put(colOutputName, gbKeys.get(i));
}
// 2. Walk through UDAF and add them to GB
String lastReduceKeyColName = null;
if (!rs.getConf().getOutputKeyColumnNames().isEmpty()) {
lastReduceKeyColName = rs.getConf().getOutputKeyColumnNames()
.get(rs.getConf().getOutputKeyColumnNames().size() - 1);
}
int numDistinctUDFs = 0;
List<ExprNodeDesc> reduceValues = rs.getConf().getValueCols();
ArrayList<AggregationDesc> aggregations = new ArrayList<AggregationDesc>();
int udafColStartPosInOriginalGB = gbInfo.gbKeys.size();
// the positions in rsColInfoLst are as follows
// --grpkey--,--distkey--,--values--
// but distUDAF may be before/after some non-distUDAF,
// i.e., their positions can be mixed.
// so for all UDAF we first check to see if it is groupby key, if not is it distinct key
// if not it should be value
List<Integer> distinctPositions = new ArrayList<>();
Map<Integer, ArrayList<ExprNodeDesc>> indexToParameter = new TreeMap<>();
for (int i = 0; i < gbInfo.udafAttrs.size(); i++) {
UDAFAttrs udafAttr = gbInfo.udafAttrs.get(i);
ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
ColumnInfo rsUDAFParamColInfo;
ExprNodeDesc udafParam;
ExprNodeDesc constantPropDistinctUDAFParam;
for (int j = 0; j < udafAttr.udafParams.size(); j++) {
int argPos = getColInfoPos(udafAttr.udafParams.get(j), gbInfo);
rsUDAFParamColInfo = rsColInfoLst.get(argPos);
String rsUDAFParamName = rsUDAFParamColInfo.getInternalName();
if (udafAttr.isDistinctUDAF && lastReduceKeyColName != null) {
rsUDAFParamName = Utilities.ReduceField.KEY.name() + "." + lastReduceKeyColName
+ ":" + numDistinctUDFs + "." + SemanticAnalyzer.getColumnInternalName(j);
}
udafParam = new ExprNodeColumnDesc(rsUDAFParamColInfo.getType(), rsUDAFParamName,
rsUDAFParamColInfo.getTabAlias(), rsUDAFParamColInfo.getIsVirtualCol());
constantPropDistinctUDAFParam = SemanticAnalyzer
.isConstantParameterInAggregationParameters(rsUDAFParamColInfo.getInternalName(),
reduceValues);
if (constantPropDistinctUDAFParam != null) {
udafParam = constantPropDistinctUDAFParam;
}
aggParameters.add(udafParam);
}
indexToParameter.put(i, aggParameters);
if(udafAttr.isDistinctUDAF) {
numDistinctUDFs++;
}
}
for(int index : indexToParameter.keySet()){
UDAFAttrs udafAttr = gbInfo.udafAttrs.get(index);
Mode udafMode = SemanticAnalyzer.groupByDescModeToUDAFMode(gbMode, udafAttr.isDistinctUDAF);
GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(udafAttr.udafEvaluator, udafMode,
indexToParameter.get(index));
aggregations.add(new AggregationDesc(udafAttr.udafName.toLowerCase(),
udaf.genericUDAFEvaluator, udaf.convertedParameters, udafAttr.isDistinctUDAF, udafMode));
if (useOriginalGBNames) {
colOutputName = gbInfo.outputColNames.get(udafColStartPosInOriginalGB + index);
} else {
colOutputName = SemanticAnalyzer.getColumnInternalName(gbKeys.size() + aggregations.size()
- 1);
}
colInfoLst.add(new ColumnInfo(colOutputName, udaf.returnType, "", false));
outputColNames.add(colOutputName);
}
Operator rsGB1 = OperatorFactory.getAndMakeChild(new GroupByDesc(gbMode, outputColNames,
gbKeys, aggregations, false, gbInfo.groupByMemoryUsage, gbInfo.memoryThreshold, null,
false, -1, numDistinctUDFs > 0), new RowSchema(colInfoLst), rs);
rsGB1.setColumnExprMap(colExprMap);
return new OpAttr("", new HashSet<Integer>(), rsGB1);
}
private static int getColInfoPos(ExprNodeDesc aggExpr, GBInfo gbInfo ) {
//first see if it is gbkeys
int gbKeyIndex = ExprNodeDescUtils.indexOf(aggExpr, gbInfo.gbKeys);
if(gbKeyIndex < 0 ) {
//then check if it is distinct key
int distinctKeyIndex = ExprNodeDescUtils.indexOf(aggExpr, gbInfo.distExprNodes);
if(distinctKeyIndex < 0) {
// lastly it should be in deDupedNonDistIrefs
int deDupValIndex = ExprNodeDescUtils.indexOf(aggExpr, gbInfo.deDupedNonDistIrefs);
assert(deDupValIndex >= 0);
return gbInfo.gbKeys.size() + gbInfo.distExprNodes.size() + deDupValIndex;
}
else {
//aggExpr is part of distinct key
return gbInfo.gbKeys.size() + distinctKeyIndex;
}
}
else {
return gbKeyIndex;
}
}
@SuppressWarnings("unchecked")
private static OpAttr genMapSideGB(OpAttr inputOpAf, GBInfo gbAttrs) throws SemanticException {
ArrayList<String> outputColNames = new ArrayList<String>();
ArrayList<ColumnInfo> colInfoLst = new ArrayList<ColumnInfo>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
Set<String> gbKeyColsAsNamesFrmIn = new HashSet<String>();
String colOutputName = null;
// 1. Build GB Keys, grouping set starting position
// 1.1 First Add original GB Keys
ArrayList<ExprNodeDesc> gbKeys = new ArrayList<ExprNodeDesc>();
for (int i = 0; i < gbAttrs.gbKeys.size(); i++) {
gbKeys.add(gbAttrs.gbKeys.get(i));
colOutputName = SemanticAnalyzer.getColumnInternalName(i);
colInfoLst.add(new ColumnInfo(colOutputName, gbAttrs.gbKeyTypes.get(i), "", false));
outputColNames.add(colOutputName);
gbKeyColsAsNamesFrmIn.add(gbAttrs.gbKeyColNamesInInput.get(i));
colExprMap.put(colOutputName, gbKeys.get(i));
}
// 1.2. Adjust GroupingSet Position, GBKeys for GroupingSet Position if
// needed. NOTE: GroupingID is added to map side GB only if we don't GrpSet
// doesn't require additional MR Jobs
int groupingSetsPosition = -1;
boolean inclGrpID = inclGrpSetInMapSide(gbAttrs);
if (inclGrpID) {
groupingSetsPosition = gbKeys.size();
addGrpSetCol(true, null, false, gbKeys, outputColNames, colInfoLst, colExprMap);
}
// 1.3. Add all distinct params
// NOTE: distinct expr can not be part of of GB key (we assume plan
// gen would have prevented it)
for (int i = 0; i < gbAttrs.distExprNodes.size(); i++) {
if (!gbKeyColsAsNamesFrmIn.contains(gbAttrs.distExprNames.get(i))) {
gbKeys.add(gbAttrs.distExprNodes.get(i));
colOutputName = SemanticAnalyzer.getColumnInternalName(gbKeys.size() - 1);
colInfoLst.add(new ColumnInfo(colOutputName, gbAttrs.distExprTypes.get(i), "", false));
outputColNames.add(colOutputName);
gbKeyColsAsNamesFrmIn.add(gbAttrs.distExprNames.get(i));
colExprMap.put(colOutputName, gbKeys.get(gbKeys.size() - 1));
}
}
// 2. Build Aggregations
ArrayList<AggregationDesc> aggregations = new ArrayList<AggregationDesc>();
for (UDAFAttrs udafAttr : gbAttrs.udafAttrs) {
Mode amode = SemanticAnalyzer.groupByDescModeToUDAFMode(GroupByDesc.Mode.HASH,
udafAttr.isDistinctUDAF);
aggregations.add(new AggregationDesc(udafAttr.udafName.toLowerCase(), udafAttr.udafEvaluator,
udafAttr.udafParams, udafAttr.isDistinctUDAF, amode));
GenericUDAFInfo udafInfo;
try {
udafInfo = SemanticAnalyzer.getGenericUDAFInfo(udafAttr.udafEvaluator, amode,
udafAttr.udafParams);
} catch (SemanticException e) {
throw new RuntimeException(e);
}
colOutputName = SemanticAnalyzer.getColumnInternalName(gbKeys.size() + aggregations.size()
- 1);
colInfoLst.add(new ColumnInfo(colOutputName, udafInfo.returnType, "", false));
outputColNames.add(colOutputName);
}
// 3. Create GB
@SuppressWarnings("rawtypes")
Operator gbOp = OperatorFactory.getAndMakeChild(new GroupByDesc(GroupByDesc.Mode.HASH,
outputColNames, gbKeys, aggregations, false, gbAttrs.groupByMemoryUsage,
gbAttrs.memoryThreshold, gbAttrs.grpSets, inclGrpID, groupingSetsPosition,
gbAttrs.containsDistinctAggr), new RowSchema(colInfoLst), inputOpAf.inputs.get(0));
// 5. Setup Expr Col Map
// NOTE: UDAF is not included in ExprColMap
gbOp.setColumnExprMap(colExprMap);
return new OpAttr("", new HashSet<Integer>(), gbOp);
}
private static void addGrpSetCol(boolean createConstantExpr, String grpSetIDExprName,
boolean addReducePrefixToColInfoName, List<ExprNodeDesc> exprLst,
List<String> outputColumnNames, List<ColumnInfo> colInfoLst,
Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
String outputColName = null;
ExprNodeDesc grpSetColExpr = null;
if (createConstantExpr) {
grpSetColExpr = new ExprNodeConstantDesc("0");
} else {
grpSetColExpr = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, grpSetIDExprName,
null, false);
}
exprLst.add(grpSetColExpr);
outputColName = SemanticAnalyzer.getColumnInternalName(exprLst.size() - 1);
outputColumnNames.add(outputColName);
String internalColName = outputColName;
if (addReducePrefixToColInfoName) {
internalColName = Utilities.ReduceField.KEY.toString() + "." + outputColName;
}
colInfoLst.add(new ColumnInfo(internalColName, grpSetColExpr.getTypeInfo(), null, true));
colExprMap.put(internalColName, grpSetColExpr);
}
/**
* Get Reduce Keys for RS following MapSide GB
*
* @param reduceKeys
* assumed to be deduped list of exprs
* @param outputKeyColumnNames
* @param colExprMap
* @return List of ExprNodeDesc of ReduceKeys
* @throws SemanticException
*/
private static ArrayList<ExprNodeDesc> getReduceKeysForRS(Operator inOp, int startPos,
int endPos, List<String> outputKeyColumnNames, boolean addOnlyOneKeyColName,
ArrayList<ColumnInfo> colInfoLst, Map<String, ExprNodeDesc> colExprMap,
boolean addEmptyTabAlias, boolean setColToNonVirtual) throws SemanticException {
ArrayList<ExprNodeDesc> reduceKeys = null;
if (endPos < 0) {
reduceKeys = new ArrayList<ExprNodeDesc>();
} else {
reduceKeys = ExprNodeDescUtils.genExprNodeDesc(inOp, startPos, endPos, addEmptyTabAlias,
setColToNonVirtual);
int outColNameIndx = startPos;
for (int i = 0; i < reduceKeys.size(); ++i) {
String outputColName = SemanticAnalyzer.getColumnInternalName(outColNameIndx);
outColNameIndx++;
if (!addOnlyOneKeyColName || i == 0) {
outputKeyColumnNames.add(outputColName);
}
// TODO: Verify if this is needed (Why can't it be always null/empty
String tabAlias = addEmptyTabAlias ? "" : null;
ColumnInfo colInfo = new ColumnInfo(Utilities.ReduceField.KEY.toString() + "."
+ outputColName, reduceKeys.get(i).getTypeInfo(), tabAlias, false);
colInfoLst.add(colInfo);
colExprMap.put(colInfo.getInternalName(), reduceKeys.get(i));
}
}
return reduceKeys;
}
/**
* Get Value Keys for RS following MapSide GB
*
* @param GroupByOperator
* MapSide GB
* @param outputKeyColumnNames
* @param colExprMap
* @return List of ExprNodeDesc of Values
* @throws SemanticException
*/
private static ArrayList<ExprNodeDesc> getValueKeysForRS(Operator inOp, int aggStartPos,
List<String> outputKeyColumnNames, ArrayList<ColumnInfo> colInfoLst,
Map<String, ExprNodeDesc> colExprMap, boolean addEmptyTabAlias, boolean setColToNonVirtual)
throws SemanticException {
List<ColumnInfo> mapGBColInfoLst = inOp.getSchema().getSignature();
ArrayList<ExprNodeDesc> valueKeys = null;
if (aggStartPos >= mapGBColInfoLst.size()) {
valueKeys = new ArrayList<ExprNodeDesc>();
} else {
valueKeys = ExprNodeDescUtils.genExprNodeDesc(inOp, aggStartPos, mapGBColInfoLst.size() - 1,
true, setColToNonVirtual);
for (int i = 0; i < valueKeys.size(); ++i) {
String outputColName = SemanticAnalyzer.getColumnInternalName(i);
outputKeyColumnNames.add(outputColName);
// TODO: Verify if this is needed (Why can't it be always null/empty
String tabAlias = addEmptyTabAlias ? "" : null;
ColumnInfo colInfo = new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "."
+ outputColName, valueKeys.get(i).getTypeInfo(), tabAlias, false);
colInfoLst.add(colInfo);
colExprMap.put(colInfo.getInternalName(), valueKeys.get(i));
}
}
return valueKeys;
}
// TODO: Implement this
private static ExprNodeDesc propConstDistUDAFParams() {
return null;
}
}