/** * (C) Copyright IBM Corp. 2010, 2015 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. *  */ package com.ibm.bi.dml.yarn.ropt; import com.ibm.bi.dml.conf.ConfigurationManager; import com.ibm.bi.dml.conf.DMLConfig; import com.ibm.bi.dml.hops.OptimizerUtils; public class YarnOptimizerUtils { public enum GridEnumType{ EQUI_GRID, EXP_GRID, MEM_EQUI_GRID, HYBRID_MEM_EXP_GRID, } /** * * @return */ public static double getRemoteMemBudgetMap(long jobLookupId) { return getRemoteMemBudgetMap(false, jobLookupId); } /** * * @return */ public static double getRemoteMemBudgetMap(boolean substractSortBuffer, long jobLookupId) { double ret = YarnClusterAnalyzer.getRemoteMaxMemoryMap(jobLookupId); if( substractSortBuffer ) ret -= YarnClusterAnalyzer.getRemoteMaxMemorySortBuffer(); return ret * OptimizerUtils.MEM_UTIL_FACTOR; } /** * * @return */ public static double getRemoteMemBudgetReduce(long jobLookupId) { double ret = YarnClusterAnalyzer.getRemoteMaxMemoryReduce(jobLookupId); return ret * OptimizerUtils.MEM_UTIL_FACTOR; } /** * Returns the number of reducers that potentially run in parallel. * This is either just the configured value (SystemML config) or * the minimum of configured value and available reduce slots. * * @param configOnly * @return */ public static int getNumReducers(boolean configOnly, long jobLookupId) { int ret = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS); if( !configOnly ) ret = Math.min(ret,YarnClusterAnalyzer.getRemoteParallelReduceTasks(jobLookupId)); return ret; } /** * * @param mb * @return */ public static long toB( long mb ) { return 1024 * 1024 * mb; } /** * * @param b * @return */ public static long toMB( long b ) { return b / (1024 * 1024); } /** * * @param minAlloc * @param maxAlloc * @param numCores * @return */ public static long computeMinContraint( long minAlloc, long maxAlloc, long numCores ) { return ((long)(Math.max(minAlloc, maxAlloc/numCores )/minAlloc)*minAlloc); } }