/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
/**
* Contains information required for memory usage monitoring.
**/
public class MemoryMonitorInfo implements Serializable {
private static final long serialVersionUID = 1L;
// Variables for LLAP hash table loading memory monitor
private boolean isLlap;
private int executorsPerNode;
private int maxExecutorsOverSubscribeMemory;
private double memoryOverSubscriptionFactor;
private long noConditionalTaskSize;
private long adjustedNoConditionalTaskSize;
private long memoryCheckInterval;
private double hashTableInflationFactor;
private long threshold;
public MemoryMonitorInfo() {
}
public MemoryMonitorInfo(boolean isLlap, int executorsPerNode, int maxExecutorsOverSubscribeMemory,
double memoryOverSubscriptionFactor, long noConditionalTaskSize, long adjustedNoConditionalTaskSize,
long memoryCheckInterval, double hashTableInflationFactor) {
this.isLlap = isLlap;
this.executorsPerNode = executorsPerNode;
this.maxExecutorsOverSubscribeMemory = maxExecutorsOverSubscribeMemory;
this.memoryOverSubscriptionFactor = memoryOverSubscriptionFactor;
this.noConditionalTaskSize = noConditionalTaskSize;
this.adjustedNoConditionalTaskSize = adjustedNoConditionalTaskSize;
this.memoryCheckInterval = memoryCheckInterval;
this.hashTableInflationFactor = hashTableInflationFactor;
this.threshold = (long) (hashTableInflationFactor * adjustedNoConditionalTaskSize);
}
public MemoryMonitorInfo(MemoryMonitorInfo memoryMonitorInfo) {
this.isLlap = memoryMonitorInfo.isLlap;
this.executorsPerNode = memoryMonitorInfo.executorsPerNode;
this.maxExecutorsOverSubscribeMemory = memoryMonitorInfo.maxExecutorsOverSubscribeMemory;
this.memoryOverSubscriptionFactor = memoryMonitorInfo.memoryOverSubscriptionFactor;
this.noConditionalTaskSize = memoryMonitorInfo.noConditionalTaskSize;
this.adjustedNoConditionalTaskSize = memoryMonitorInfo.adjustedNoConditionalTaskSize;
this.memoryCheckInterval = memoryMonitorInfo.memoryCheckInterval;
this.hashTableInflationFactor = memoryMonitorInfo.hashTableInflationFactor;
this.threshold = memoryMonitorInfo.threshold;
}
public int getExecutorsPerNode() {
return executorsPerNode;
}
public void setExecutorsPerNode(final int executorsPerNode) {
this.executorsPerNode = executorsPerNode;
}
public int getMaxExecutorsOverSubscribeMemory() {
return maxExecutorsOverSubscribeMemory;
}
public void setMaxExecutorsOverSubscribeMemory(final int maxExecutorsOverSubscribeMemory) {
this.maxExecutorsOverSubscribeMemory = maxExecutorsOverSubscribeMemory;
}
public double getMemoryOverSubscriptionFactor() {
return memoryOverSubscriptionFactor;
}
public void setMemoryOverSubscriptionFactor(final double memoryOverSubscriptionFactor) {
this.memoryOverSubscriptionFactor = memoryOverSubscriptionFactor;
}
public long getNoConditionalTaskSize() {
return noConditionalTaskSize;
}
public void setNoConditionalTaskSize(final long noConditionalTaskSize) {
this.noConditionalTaskSize = noConditionalTaskSize;
}
public long getAdjustedNoConditionalTaskSize() {
return adjustedNoConditionalTaskSize;
}
public void setAdjustedNoConditionalTaskSize(final long adjustedNoConditionalTaskSize) {
this.adjustedNoConditionalTaskSize = adjustedNoConditionalTaskSize;
}
public long getMemoryCheckInterval() {
return memoryCheckInterval;
}
public void setMemoryCheckInterval(final long memoryCheckInterval) {
this.memoryCheckInterval = memoryCheckInterval;
}
public double getHashTableInflationFactor() {
return hashTableInflationFactor;
}
public void setHashTableInflationFactor(final double hashTableInflationFactor) {
this.hashTableInflationFactor = hashTableInflationFactor;
}
public long getThreshold() {
return threshold;
}
public void setLlap(final boolean llap) {
isLlap = llap;
}
public boolean isLlap() {
return isLlap;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{");
sb.append(" isLlap: ").append(isLlap);
sb.append(" executorsPerNode: ").append(executorsPerNode);
sb.append(" maxExecutorsOverSubscribeMemory: ").append(maxExecutorsOverSubscribeMemory);
sb.append(" memoryOverSubscriptionFactor: ").append(memoryOverSubscriptionFactor);
sb.append(" memoryCheckInterval: ").append(memoryCheckInterval);
sb.append(" noConditionalTaskSize: ").append(noConditionalTaskSize);
sb.append(" adjustedNoConditionalTaskSize: ").append(adjustedNoConditionalTaskSize);
sb.append(" hashTableInflationFactor: ").append(hashTableInflationFactor);
sb.append(" threshold: ").append(threshold);
sb.append(" }");
return sb.toString();
}
public boolean doMemoryMonitoring() {
return isLlap && hashTableInflationFactor > 0.0d && noConditionalTaskSize > 0 &&
memoryCheckInterval > 0;
}
public long getEffectiveThreshold(final long maxMemoryPerExecutor) {
// guard against poor configuration of noconditional task size. We let hash table grow till 2/3'rd memory
// available for container/executor
return (long) Math.max(threshold, (2.0 / 3.0) * maxMemoryPerExecutor);
}
}