/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.plan;
import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.BucketMatcher;
import org.apache.hadoop.hive.ql.exec.Operator;
/**
* MapredLocalWork.
*
*/
@Explain(displayName = "Map Reduce Local Work")
public class MapredLocalWork implements Serializable {
private static final long serialVersionUID = 1L;
private LinkedHashMap<String, Operator<? extends Serializable>> aliasToWork;
private LinkedHashMap<String, FetchWork> aliasToFetchWork;
private boolean inputFileChangeSensitive;
private BucketMapJoinContext bucketMapjoinContext;
private String tmpFileURI;
private String stageID;
private List<Operator<? extends Serializable>> dummyParentOp ;
public MapredLocalWork() {
}
public MapredLocalWork(
final LinkedHashMap<String, Operator<? extends Serializable>> aliasToWork,
final LinkedHashMap<String, FetchWork> aliasToFetchWork) {
this.aliasToWork = aliasToWork;
this.aliasToFetchWork = aliasToFetchWork;
}
public MapredLocalWork(MapredLocalWork clone){
this.tmpFileURI = clone.tmpFileURI;
this.inputFileChangeSensitive=clone.inputFileChangeSensitive;
}
public void setDummyParentOp(List<Operator<? extends Serializable>> op){
this.dummyParentOp=op;
}
public List<Operator<? extends Serializable>> getDummyParentOp(){
return this.dummyParentOp;
}
@Explain(displayName = "Alias -> Map Local Operator Tree")
public LinkedHashMap<String, Operator<? extends Serializable>> getAliasToWork() {
return aliasToWork;
}
public String getStageID() {
return stageID;
}
public void setStageID(String stageID) {
this.stageID = stageID;
}
public void setAliasToWork(
final LinkedHashMap<String, Operator<? extends Serializable>> aliasToWork) {
this.aliasToWork = aliasToWork;
}
/**
* @return the aliasToFetchWork
*/
@Explain(displayName = "Alias -> Map Local Tables")
public LinkedHashMap<String, FetchWork> getAliasToFetchWork() {
return aliasToFetchWork;
}
/**
* @param aliasToFetchWork
* the aliasToFetchWork to set
*/
public void setAliasToFetchWork(
final LinkedHashMap<String, FetchWork> aliasToFetchWork) {
this.aliasToFetchWork = aliasToFetchWork;
}
public boolean getInputFileChangeSensitive() {
return inputFileChangeSensitive;
}
public void setInputFileChangeSensitive(boolean inputFileChangeSensitive) {
this.inputFileChangeSensitive = inputFileChangeSensitive;
}
public void deriveExplainAttributes() {
if (bucketMapjoinContext != null) {
bucketMapjoinContext.deriveBucketMapJoinMapping();
}
for (FetchWork fetchWork : aliasToFetchWork.values()) {
if (fetchWork.getTblDesc() == null) {
continue;
}
PlanUtils.configureInputJobPropertiesForStorageHandler(
fetchWork.getTblDesc());
}
}
@Explain(displayName = "Bucket Mapjoin Context", normalExplain = false)
public BucketMapJoinContext getBucketMapjoinContext() {
return bucketMapjoinContext;
}
public void setBucketMapjoinContext(BucketMapJoinContext bucketMapjoinContext) {
this.bucketMapjoinContext = bucketMapjoinContext;
}
public void setTmpFileURI(String tmpFileURI) {
this.tmpFileURI = tmpFileURI;
}
public String getTmpFileURI() {
return tmpFileURI;
}
public static class BucketMapJoinContext implements Serializable {
private static final long serialVersionUID = 1L;
// used for bucket map join
private LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketFileNameMapping;
private String mapJoinBigTableAlias;
private Class<? extends BucketMatcher> bucketMatcherClass;
private LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketBaseFileNameMapping;
private LinkedHashMap<String, Integer> bucketFileNameMapping;
public void setMapJoinBigTableAlias(String bigTableAlias) {
this.mapJoinBigTableAlias = bigTableAlias;
}
public void deriveBucketMapJoinMapping() {
if (aliasBucketFileNameMapping != null) {
Iterator<Entry<String, LinkedHashMap<String, ArrayList<String>>>> iter =
aliasBucketFileNameMapping.entrySet().iterator();
aliasBucketBaseFileNameMapping = new LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>>();
while (iter.hasNext()) {
Entry<String, LinkedHashMap<String, ArrayList<String>>> old = iter.next();
LinkedHashMap<String, ArrayList<String>> newBucketBaseFileNameMapping = new LinkedHashMap<String, ArrayList<String>>();
Iterator<Entry<String, ArrayList<String>>> oldAliasFileNameMappingIter = old.getValue().entrySet().iterator();
while (oldAliasFileNameMappingIter.hasNext()) {
//For a give table and its bucket full file path list, only keep the base file name (remove file path etc).
//And put the new list into the new mapping.
Entry<String, ArrayList<String>> oldTableBucketFileNames = oldAliasFileNameMappingIter.next();
ArrayList<String> oldTableBucketNames = oldTableBucketFileNames.getValue();
ArrayList<String> newTableBucketFileBaseName = new ArrayList<String> (oldTableBucketNames.size());
//for each bucket file, only keep its base files and store into a new list.
if (oldTableBucketNames != null) {
for (String bucketFName : oldTableBucketNames) {
newTableBucketFileBaseName.add(getBaseFileName(bucketFName));
}
}
String bigTblBucketFileName = getBaseFileName(oldTableBucketFileNames.getKey());
if(newBucketBaseFileNameMapping.containsKey(bigTblBucketFileName)) {
String fullPath = oldTableBucketFileNames.getKey();
String dir = getBaseFileName(fullPath.substring(0, fullPath.lastIndexOf(bigTblBucketFileName)));
bigTblBucketFileName = dir + File.separator + bigTblBucketFileName;
}
//put the new mapping
newBucketBaseFileNameMapping.put(bigTblBucketFileName, newTableBucketFileBaseName);
}
String tableAlias = old.getKey();
aliasBucketBaseFileNameMapping.put(tableAlias, newBucketBaseFileNameMapping);
}
}
}
private String getBaseFileName (String path) {
try {
return ((new Path(path)).getName());
} catch (Exception ex) {
// This could be due to either URI syntax error or File constructor
// illegal arg; we don't really care which one it is.
return path;
}
}
public String getMapJoinBigTableAlias() {
return mapJoinBigTableAlias;
}
public Class<? extends BucketMatcher> getBucketMatcherClass() {
return bucketMatcherClass;
}
public void setBucketMatcherClass(
Class<? extends BucketMatcher> bucketMatcherClass) {
this.bucketMatcherClass = bucketMatcherClass;
}
@Explain(displayName = "Alias Bucket File Name Mapping", normalExplain = false)
public LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> getAliasBucketFileNameMapping() {
return aliasBucketFileNameMapping;
}
public void setAliasBucketFileNameMapping(
LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketFileNameMapping) {
this.aliasBucketFileNameMapping = aliasBucketFileNameMapping;
}
@Override
public String toString() {
if (aliasBucketFileNameMapping != null) {
return "Mapping:" + aliasBucketFileNameMapping.toString();
} else {
return "";
}
}
@Explain(displayName = "Alias Bucket Base File Name Mapping", normalExplain = false)
public LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> getAliasBucketBaseFileNameMapping() {
return aliasBucketBaseFileNameMapping;
}
public void setAliasBucketBaseFileNameMapping(
LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketBaseFileNameMapping) {
this.aliasBucketBaseFileNameMapping = aliasBucketBaseFileNameMapping;
}
@Explain(displayName = "Alias Bucket Output File Name Mapping", normalExplain = false)
public LinkedHashMap<String, Integer> getBucketFileNameMapping() {
return bucketFileNameMapping;
}
public void setBucketFileNameMapping(LinkedHashMap<String, Integer> bucketFileNameMapping) {
this.bucketFileNameMapping = bucketFileNameMapping;
}
}
}