/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.optimizer;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Stack;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.BucketMapJoinContext;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
/**
* Operator factory for MapJoin processing.
*/
public final class MapJoinFactory {
public static int getPositionParent(AbstractMapJoinOperator<? extends MapJoinDesc> op,
Stack<Node> stack) {
int pos = 0;
int size = stack.size();
assert size >= 2 && stack.get(size - 1) == op;
Operator<? extends OperatorDesc> parent =
(Operator<? extends OperatorDesc>) stack.get(size - 2);
List<Operator<? extends OperatorDesc>> parOp = op.getParentOperators();
pos = parOp.indexOf(parent);
assert pos < parOp.size();
return pos;
}
/**
* MapJoin processor.
* The user can specify a mapjoin hint to specify that the input should be processed as a
* mapjoin instead of map-reduce join. If hive.auto.convert.join is set to true, the
* user need not specify the hint explicitly, but hive will automatically process the joins
* as a mapjoin whenever possible. However, a join can only be processed as a bucketized
* map-side join or a sort merge join, if the user has provided the hint explicitly. This
* will be fixed as part of HIVE-3433, and eventually, we should remove support for mapjoin
* hint.
* However, currently, the mapjoin hint is processed as follows:
* A mapjoin will have 'n' parents for a n-way mapjoin, and therefore the mapjoin operator
* will be encountered 'n' times (one for each parent). Since a reduceSink operator is not
* allowed before a mapjoin, the task for the mapjoin will always be a root task. The task
* corresponding to the mapjoin is converted to a root task when the operator is encountered
* for the first time. When the operator is encountered subsequently, the current task is
* merged with the root task for the mapjoin. Note that, it is possible that the map-join task
* may be performed as a bucketized map-side join (or sort-merge join), the map join operator
* is enhanced to contain the bucketing info. when it is encountered.
*/
private static class TableScanMapJoinProcessor implements NodeProcessor {
public static void setupBucketMapJoinInfo(MapWork plan,
AbstractMapJoinOperator<? extends MapJoinDesc> currMapJoinOp) {
if (currMapJoinOp != null) {
Map<String, Map<String, List<String>>> aliasBucketFileNameMapping =
currMapJoinOp.getConf().getAliasBucketFileNameMapping();
if (aliasBucketFileNameMapping != null) {
MapredLocalWork localPlan = plan.getMapRedLocalWork();
if (localPlan == null) {
if (currMapJoinOp instanceof SMBMapJoinOperator) {
localPlan = ((SMBMapJoinOperator) currMapJoinOp).getConf().getLocalWork();
}
} else {
// local plan is not null, we want to merge it into SMBMapJoinOperator's local work
if (currMapJoinOp instanceof SMBMapJoinOperator) {
MapredLocalWork smbLocalWork = ((SMBMapJoinOperator) currMapJoinOp).getConf()
.getLocalWork();
if (smbLocalWork != null) {
localPlan.getAliasToFetchWork().putAll(smbLocalWork.getAliasToFetchWork());
localPlan.getAliasToWork().putAll(smbLocalWork.getAliasToWork());
}
}
}
if (localPlan == null) {
return;
}
if (currMapJoinOp instanceof SMBMapJoinOperator) {
plan.setMapRedLocalWork(null);
((SMBMapJoinOperator) currMapJoinOp).getConf().setLocalWork(localPlan);
} else {
plan.setMapRedLocalWork(localPlan);
}
BucketMapJoinContext bucketMJCxt = new BucketMapJoinContext();
localPlan.setBucketMapjoinContext(bucketMJCxt);
bucketMJCxt.setAliasBucketFileNameMapping(aliasBucketFileNameMapping);
bucketMJCxt.setBucketFileNameMapping(
currMapJoinOp.getConf().getBigTableBucketNumMapping());
localPlan.setInputFileChangeSensitive(true);
bucketMJCxt.setMapJoinBigTableAlias(currMapJoinOp.getConf().getBigTableAlias());
bucketMJCxt
.setBucketMatcherClass(org.apache.hadoop.hive.ql.exec.DefaultBucketMatcher.class);
bucketMJCxt.setBigTablePartSpecToFileMapping(
currMapJoinOp.getConf().getBigTablePartSpecToFileMapping());
// BucketizedHiveInputFormat should be used for either sort merge join or bucket map join
if ((currMapJoinOp instanceof SMBMapJoinOperator)
|| (currMapJoinOp.getConf().isBucketMapJoin())) {
plan.setUseBucketizedHiveInputFormat(true);
}
}
}
}
/**
* Initialize the current plan by adding it to root tasks. Since a reduce sink
* cannot be present before a mapjoin, and the mapjoin operator is encountered
* for the first time, the task corresposding to the mapjoin is added to the
* root tasks.
*
* @param op
* the map join operator encountered
* @param opProcCtx
* processing context
* @param pos
* position of the parent
*/
private static void initMapJoinPlan(AbstractMapJoinOperator<? extends MapJoinDesc> op,
Task<? extends Serializable> currTask,
GenMRProcContext opProcCtx, boolean local)
throws SemanticException {
// The map is overloaded to keep track of mapjoins also
opProcCtx.getOpTaskMap().put(op, currTask);
TableScanOperator currTopOp = opProcCtx.getCurrTopOp();
String currAliasId = opProcCtx.getCurrAliasId();
GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, local, opProcCtx);
}
/**
* Merge the current task with the task for the current mapjoin. The mapjoin operator
* has already been encountered.
*
* @param op
* operator being processed
* @param oldTask
* the old task for the current mapjoin
* @param opProcCtx
* processing context
* @param pos
* position of the parent in the stack
*/
private static void joinMapJoinPlan(Task<? extends Serializable> oldTask,
GenMRProcContext opProcCtx, boolean local)
throws SemanticException {
TableScanOperator currTopOp = opProcCtx.getCurrTopOp();
GenMapRedUtils.mergeInput(currTopOp, opProcCtx, oldTask, local);
}
/*
* The mapjoin operator will be encountered many times (n times for a n-way join). Since a
* reduceSink operator is not allowed before a mapjoin, the task for the mapjoin will always
* be a root task. The task corresponding to the mapjoin is converted to a root task when the
* operator is encountered for the first time. When the operator is encountered subsequently,
* the current task is merged with the root task for the mapjoin. Note that, it is possible
* that the map-join task may be performed as a bucketized map-side join (or sort-merge join),
* the map join operator is enhanced to contain the bucketing info. when it is encountered.
*/
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
AbstractMapJoinOperator<MapJoinDesc> mapJoin = (AbstractMapJoinOperator<MapJoinDesc>) nd;
GenMRProcContext ctx = (GenMRProcContext) procCtx;
// find the branch on which this processor was invoked
int pos = getPositionParent(mapJoin, stack);
Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx = ctx
.getMapCurrCtx();
GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(pos));
Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
MapredWork currPlan = (MapredWork) currTask.getWork();
String currAliasId = mapredCtx.getCurrAliasId();
HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
ctx.getOpTaskMap();
Task<? extends Serializable> oldTask = opTaskMap.get(mapJoin);
ctx.setCurrAliasId(currAliasId);
ctx.setCurrTask(currTask);
// If we are seeing this mapjoin for the first time, initialize the plan.
// If we are seeing this mapjoin for the second or later time then atleast one of the
// branches for this mapjoin have been encounered. Join the plan with the plan created
// the first time.
boolean local = pos != mapJoin.getConf().getPosBigTable();
if (oldTask == null) {
assert currPlan.getReduceWork() == null;
initMapJoinPlan(mapJoin, currTask, ctx, local);
} else {
// The current plan can be thrown away after being merged with the
// original plan
joinMapJoinPlan(oldTask, ctx, local);
ctx.setCurrTask(currTask = oldTask);
}
MapredWork plan = (MapredWork) currTask.getWork();
setupBucketMapJoinInfo(plan.getMapWork(), mapJoin);
mapCurrCtx.put(mapJoin, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrAliasId()));
// local aliases need not to hand over context further
return !local;
}
}
public static NodeProcessor getTableScanMapJoin() {
return new TableScanMapJoinProcessor();
}
private MapJoinFactory() {
// prevent instantiation
}
}