ClusterConfigManager.java

/***************************************************************************
 * Copyright (c) 2012-2015 VMware, Inc. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ***************************************************************************/
package com.vmware.bdd.manager;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.collections.MapUtils;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.vmware.aurora.global.Configuration;
import com.vmware.aurora.util.HbaseRegionServerOptsUtil;
import com.vmware.aurora.vc.DiskSpec.AllocationType;
import com.vmware.bdd.apitypes.ClusterCreate;
import com.vmware.bdd.apitypes.ClusterNetConfigInfo;
import com.vmware.bdd.apitypes.Datastore.DatastoreType;
import com.vmware.bdd.apitypes.DiskSplitPolicy;
import com.vmware.bdd.apitypes.IpBlock;
import com.vmware.bdd.apitypes.IpConfigInfo;
import com.vmware.bdd.apitypes.LatencyPriority;
import com.vmware.bdd.apitypes.NetConfigInfo.NetTrafficType;
import com.vmware.bdd.apitypes.NetworkAdd;
import com.vmware.bdd.apitypes.NodeGroupCreate;
import com.vmware.bdd.apitypes.NodeRead;
import com.vmware.bdd.apitypes.PlacementPolicy;
import com.vmware.bdd.apitypes.PlacementPolicy.GroupAssociation;
import com.vmware.bdd.apitypes.PlacementPolicy.GroupRacks;
import com.vmware.bdd.apitypes.PlacementPolicy.GroupRacks.GroupRacksType;
import com.vmware.bdd.apitypes.RackInfo;
import com.vmware.bdd.apitypes.StorageRead;
import com.vmware.bdd.apitypes.StorageRead.DiskScsiControllerType;
import com.vmware.bdd.apitypes.TopologyType;
import com.vmware.bdd.entity.ClusterEntity;
import com.vmware.bdd.entity.IpBlockEntity;
import com.vmware.bdd.entity.NetworkEntity;
import com.vmware.bdd.entity.NodeGroupAssociation;
import com.vmware.bdd.entity.NodeGroupEntity;
import com.vmware.bdd.exception.BddException;
import com.vmware.bdd.exception.ClusterConfigException;
import com.vmware.bdd.exception.UniqueConstraintViolationException;
import com.vmware.bdd.exception.VcProviderException;
import com.vmware.bdd.manager.intf.IClusterEntityManager;
import com.vmware.bdd.service.IClusteringService;
import com.vmware.bdd.service.resmgmt.IDatastoreService;
import com.vmware.bdd.service.resmgmt.INetworkService;
import com.vmware.bdd.service.resmgmt.INodeTemplateService;
import com.vmware.bdd.service.resmgmt.IResourcePoolService;
import com.vmware.bdd.software.mgmt.plugin.exception.ValidationException;
import com.vmware.bdd.software.mgmt.plugin.intf.SoftwareManager;
import com.vmware.bdd.software.mgmt.plugin.model.ClusterBlueprint;
import com.vmware.bdd.software.mgmt.plugin.model.HadoopStack;
import com.vmware.bdd.software.mgmt.plugin.model.NodeGroupInfo;
import com.vmware.bdd.specpolicy.CommonClusterExpandPolicy;
import com.vmware.bdd.spectypes.HadoopRole;
import com.vmware.bdd.spectypes.VcCluster;
import com.vmware.bdd.utils.CommonUtil;
import com.vmware.bdd.utils.ConfigInfo;
import com.vmware.bdd.utils.Constants;
import com.vmware.bdd.utils.InfrastructureConfigUtils;
import com.vmware.bdd.utils.VcVmUtil;

public class ClusterConfigManager {
   private static final long serialVersionUID = 1L;
   private static final Logger logger = Logger
         .getLogger(ClusterConfigManager.class);

   private IResourcePoolService rpMgr;
   private INetworkService networkMgr;
   private RackInfoManager rackInfoMgr;
   private IDatastoreService datastoreMgr;
   private IClusterEntityManager clusterEntityMgr;
   private IClusteringService clusteringService;
   @Autowired
   private INodeTemplateService nodeTemplateService;

   private SoftwareManagerCollector softwareManagerCollector;

   private static final String HTTP_PROXY = "serengeti.http_proxy";
   private static final String NO_PROXY = "serengeti.no_proxy";
   private static final String ELASTIC_RUNTIME_AUTOMATION_ENABLE =
         "elastic_runtime.automation.enable";
   private final String httpProxy = Configuration
         .getString(HTTP_PROXY.toString(), "");
   private final String noProxy = Configuration.getStrings(NO_PROXY.toString(), "");
   private final boolean automationEnable = Configuration.getBoolean(
         ELASTIC_RUNTIME_AUTOMATION_ENABLE, false);

   public IDatastoreService getDatastoreMgr() {
      return datastoreMgr;
   }

   public void setDatastoreMgr(IDatastoreService datastoreMgr) {
      this.datastoreMgr = datastoreMgr;
   }

   public RackInfoManager getRackInfoMgr() {
      return rackInfoMgr;
   }

   public void setRackInfoMgr(RackInfoManager rackInfoMgr) {
      this.rackInfoMgr = rackInfoMgr;
   }

   public IResourcePoolService getRpMgr() {
      return rpMgr;
   }

   public void setRpMgr(IResourcePoolService rpMgr) {
      this.rpMgr = rpMgr;
   }

   public INetworkService getNetworkMgr() {
      return networkMgr;
   }

   public void setNetworkMgr(INetworkService networkMgr) {
      this.networkMgr = networkMgr;
   }

   public IClusterEntityManager getClusterEntityMgr() {
      return clusterEntityMgr;
   }

   @Autowired
   public void setClusterEntityMgr(IClusterEntityManager clusterEntityMgr) {
      this.clusterEntityMgr = clusterEntityMgr;
   }

   public IClusteringService getClusteringService() {
      return clusteringService;
   }

   @Autowired
   public void setClusteringService(IClusteringService clusteringService) {
      this.clusteringService = clusteringService;
   }

   @Autowired
   public void setSoftwareManagerCollector(
         SoftwareManagerCollector softwareManagerCollector) {
      this.softwareManagerCollector = softwareManagerCollector;
   }

   private void applyInfraChanges(ClusterCreate cluster,
         ClusterBlueprint blueprint) {
      cluster.setConfiguration(blueprint.getConfiguration());
      sortNodeGroups(cluster, blueprint);
      // as we've sorted node groups, so here we can assume node group are in same location in the array.
      for (int i = 0; i < blueprint.getNodeGroups().size(); i++) {
         NodeGroupInfo group = blueprint.getNodeGroups().get(i);
         NodeGroupCreate groupCreate = cluster.getNodeGroups()[i];
         groupCreate.setConfiguration(group.getConfiguration());
         groupCreate.setRoles(group.getRoles());
         groupCreate.setInstanceType(group.getInstanceType());
         groupCreate.setPlacementPolicies(group.getPlacement());
         if (groupCreate.getStorage() == null) {
            groupCreate.setStorage(new StorageRead());
         }
         groupCreate.getStorage().setSizeGB(group.getStorageSize());
      }
      cluster.setExternalHDFS(blueprint.getExternalHDFS());
      cluster.setExternalMapReduce(blueprint.getExternalMapReduce());
   }

   private void sortNodeGroups(ClusterCreate cluster, ClusterBlueprint blueprint) {
      NodeGroupCreate[] sortedGroups =
            new NodeGroupCreate[cluster.getNodeGroups().length];
      for (int i = 0; i < blueprint.getNodeGroups().size(); i++) {
         NodeGroupInfo groupInfo = blueprint.getNodeGroups().get(i);
         if (cluster.getNodeGroups()[i].getName().equals(groupInfo.getName())) {
            // to save query time
            sortedGroups[i] = cluster.getNodeGroups()[i];
         }
         sortedGroups[i] = cluster.getNodeGroup(groupInfo.getName());
      }
      cluster.setNodeGroups(sortedGroups);
   }

   @Transactional
   public ClusterEntity createClusterConfig(ClusterCreate cluster) {
      String name = cluster.getName();
      if (name == null || name.isEmpty()) {
         throw ClusterConfigException.CLUSTER_NAME_MISSING();
      }

      List<String> failedMsgList = new ArrayList<String>();
      List<String> warningMsgList = new ArrayList<String>();

      String appManager = cluster.getAppManager();
      if (appManager == null) {
         appManager = Constants.IRONFAN;
      }
      SoftwareManager softwareManager = getSoftwareManager(appManager);
      HadoopStack stack =
            filterDistroFromAppManager(softwareManager, cluster.getDistro());
      if (cluster.getDistro() == null || stack == null) {
         throw BddException.INVALID_PARAMETER("distro", cluster.getDistro());
      }
      // only check roles validity in server side, but not in CLI and GUI, because roles info exist in server side.
      ClusterBlueprint blueprint = cluster.toBlueprint();
      try {
         softwareManager.validateBlueprint(cluster.toBlueprint());
         cluster.validateClusterCreate(failedMsgList, warningMsgList);
      } catch (ValidationException e) {
         failedMsgList.addAll(e.getFailedMsgList());
         warningMsgList.addAll(e.getWarningMsgList());
      }

      if (!failedMsgList.isEmpty()) {
         throw ClusterConfigException.INVALID_SPEC(failedMsgList);
      }

      if (!validateRacksInfo(cluster, failedMsgList)) {
         throw ClusterConfigException.INVALID_PLACEMENT_POLICIES(failedMsgList);
      }

      String localRepoURL = cluster.getLocalRepoURL();
      if (!CommonUtil.isBlank(localRepoURL)
            && !validateLocalRepoURL(localRepoURL)) {
         throw ClusterConfigException.INVALID_LOCAL_REPO_URL(failedMsgList);
      }

      try {
         ClusterEntity entity = clusterEntityMgr.findByName(name);
         if (entity != null) {
            logger.info("can not create cluster " + name
                  + ", which is already existed.");
            throw BddException.ALREADY_EXISTS("Cluster", name);
         }

         updateInfrastructure(cluster, softwareManager, blueprint);
         // persist cluster config
         logger.debug("begin to add cluster config for " + name);
         Gson gson = new Gson();
         ClusterEntity clusterEntity = new ClusterEntity(name);
         clusterEntity.setAppManager(cluster.getAppManager());
         clusterEntity.setDistro(cluster.getDistro());
         clusterEntity.setDistroVendor(cluster.getDistroVendor());
         clusterEntity.setDistroVersion(cluster.getDistroVersion());
         clusterEntity.setStartAfterDeploy(true);
         clusterEntity.setPassword(cluster.getPassword());
         clusterEntity.setTemplateId(this.nodeTemplateService.getNodeTemplateIdByName(cluster.getTemplateName()));

         // set cluster version
         clusterEntity.setVersion(clusterEntityMgr.getServerVersion());

         if (softwareManager.containsComputeOnlyNodeGroups(blueprint)) {
            clusterEntity.setAutomationEnable(automationEnable);
         } else {
            clusterEntity.setAutomationEnable(null);
         }
         clusterEntity.setVhmMinNum(-1);
         clusterEntity.setVhmMaxNum(-1);

         if (cluster.getRpNames() != null && cluster.getRpNames().size() > 0) {
            logger.debug("resource pool " + cluster.getRpNames()
                  + " specified for cluster " + name);
            clusterEntity.setVcRpNameList(cluster.getRpNames());
         } else {
            logger.debug("no resource pool name specified, use global configuration.");
         }
         if (cluster.getDsNames() != null && !cluster.getDsNames().isEmpty()) {
            logger.debug("datastore " + cluster.getDsNames()
                  + " specified for cluster " + name);
            clusterEntity.setVcDatastoreNameList(cluster.getDsNames());
         } else {
            logger.debug("no datastore name specified, use global configuration.");
         }

         clusterEntity.setNetworkConfig(validateAndConvertNetNamesToNetConfigs(
               cluster.getNetworkConfig(), cluster.getDistroVendor()
                     .equalsIgnoreCase(Constants.MAPR_VENDOR)));
         clusterEntity.setVhmJobTrackerPort("50030");
         if (cluster.getConfiguration() != null
               && cluster.getConfiguration().size() > 0) {
            clusterEntity.setHadoopConfig((new Gson()).toJson(cluster
                  .getConfiguration()));

            updateVhmJobTrackerPort(cluster, clusterEntity);
         }


         setInfraConfig(cluster, clusterEntity);

         setAdvancedProperties(cluster.getExternalHDFS(),
               cluster.getExternalMapReduce(), localRepoURL,
               cluster.getExternalNamenode(),
               cluster.getExternalSecondaryNamenode(),
               cluster.getExternalDatanodes(),
               cluster.getClusterCloneType(),
               clusterEntity);
         NodeGroupCreate[] groups = cluster.getNodeGroups();
         if (groups != null && groups.length > 0) {
            clusterEntity.setNodeGroups(convertNodeGroupsToEntities(gson,
                  clusterEntity, cluster.getDistro(), groups,
                  cluster.isValidateConfig()));

            //make sure memory size is no less than MIN_MEM_SIZE
            validateMemorySize(clusterEntity.getNodeGroups(), failedMsgList);
            if (!failedMsgList.isEmpty()) {
               throw ClusterConfigException.INVALID_SPEC(failedMsgList);
            }
         }
         //Set HBASE_REGISIONSERVER_OPTS for ironfan hbase_regionserer whose latencySensitivity is set to HIGH
         if(!CommonUtil.isBlank(cluster.getAppManager()) && Constants.IRONFAN.equals(cluster.getAppManager()))
            for(NodeGroupEntity group : clusterEntity.getNodeGroups()){
               String groupRoles = group.getRoles();
               if((group.getLatencySensitivity() == LatencyPriority.HIGH)
                     && ((groupRoles.contains(HadoopRole.HBASE_REGIONSERVER_ROLE.toString())))){
                     setHbaseRegionServerOpts(cluster, group);
                     if (cluster.getConfiguration() != null
                           && cluster.getConfiguration().size() > 0) {
                        clusterEntity.setHadoopConfig((new Gson()).toJson(cluster
                              .getConfiguration()));
                     }
                     break;
               }
            }


         if (cluster.getTopologyPolicy() == null) {
            clusterEntity.setTopologyPolicy(TopologyType.NONE);
         } else {
            clusterEntity.setTopologyPolicy(cluster.getTopologyPolicy());
         }

         if (clusterEntity.getTopologyPolicy() == TopologyType.HVE) {
            boolean hveSupported = false;
            if (clusterEntity.getDistro() != null) {
               HadoopStack hadoopStack =
                     filterDistroFromAppManager(softwareManager,
                           clusterEntity.getDistro());
               if (hadoopStack != null) {
                  hveSupported = hadoopStack.isHveSupported();
               }
            }
            if (!hveSupported) {
               throw ClusterConfigException.INVALID_TOPOLOGY_POLICY(
                     clusterEntity.getTopologyPolicy(),
                     "current Hadoop distribution does not support HVE.");
            }
         }

         clusterEntityMgr.insert(clusterEntity);
         logger.debug("finished to add cluster config for " + name);
         return clusterEntity;
      } catch (UniqueConstraintViolationException ex) {
         logger.info("can not create cluster " + name
               + ", which is already existed.");
         throw BddException.ALREADY_EXISTS(ex, "Cluster", name);
      }
   }

   /**
    * Set the HBASE_REGIONSERVER_OPTS for hbase_regionserver node when the nodegroup's latencySensitivity is High
    * @param cluster
    * @param ng
    */
   private void setHbaseRegionServerOpts(ClusterCreate cluster, NodeGroupEntity ng) {
      Map<String, Object> conf = cluster.getConfiguration();
      if (conf == null) {
         conf = new HashMap<String, Object>();
         cluster.setConfiguration(conf);
      }

      Map<String, Object> hbase = (Map<String, Object>) conf.get("hbase");
      if (hbase == null) {
         hbase = new HashMap<String, Object>();
         conf.put("hbase", hbase);
      }

      Map<String, Object> hbaseEnv = (Map<String, Object>) hbase.get("hbase-env.sh");
      if (hbaseEnv == null) {
         hbaseEnv = new HashMap<String, Object>();
         hbase.put("hbase-env.sh", hbaseEnv);
      }

      if (hbaseEnv.get("HBASE_REGIONSERVER_OPTS") == null) {
         Gson gson = new Gson();
         int roleNums = gson.fromJson(ng.getRoles(), List.class).size();
         hbaseEnv.put("HBASE_REGIONSERVER_OPTS", HbaseRegionServerOptsUtil
                     .getIronfanHbaseRegionServerStringParameter(
                           ng.getMemorySize(), roleNums));
         logger.info("hbase-env.sh" + hbase.get("HBASE_REGIONSERVER_OPTS"));
      }
   }

   private void setInfraConfig(ClusterCreate cluster, ClusterEntity clusterEntity) {
      Map<String, Map<String, String>> infraConfigs = cluster.getInfrastructure_config();

      if(MapUtils.isNotEmpty(infraConfigs)) {
         clusterEntity.setInfraConfig(InfrastructureConfigUtils.write(infraConfigs));
      }
   }

   private void setAdvancedProperties(String externalHDFS,
         String externalMapReduce, String localRepoURL,
         String externalNamenode, String externalSecondaryNamenode,
         Set<String> externalDatanodes, String clusterCloneType,ClusterEntity clusterEntity) {
      if (!CommonUtil.isBlank(externalHDFS)
            || !CommonUtil.isBlank(externalMapReduce)
            || !CommonUtil.isBlank(localRepoURL)
            || !CommonUtil.isBlank(externalNamenode)
            || !CommonUtil.isBlank(externalSecondaryNamenode)
            || !CommonUtil.isBlank(clusterCloneType)
            || (externalDatanodes != null && !externalDatanodes.isEmpty())) {
         Map<String, Object> advancedProperties = new HashMap<String, Object>();
         advancedProperties.put("ExternalHDFS", externalHDFS);
         advancedProperties.put("ExternalMapReduce", externalMapReduce);
         advancedProperties.put("LocalRepoURL", localRepoURL);
         advancedProperties.put("ExternalNamenode", externalNamenode);
         advancedProperties.put("ExternalSecondaryNamenode", externalSecondaryNamenode);
         advancedProperties.put("ExternalDatanodes", externalDatanodes);
         advancedProperties.put("ClusterCloneType", clusterCloneType);
         Gson g = new Gson();
         clusterEntity.setAdvancedProperties(g.toJson(advancedProperties));
      }
   }

   public HadoopStack filterDistroFromAppManager(
         SoftwareManager softwareManager, String distroName) {
      List<HadoopStack> hadoopStacks = softwareManager.getSupportedStacks();
      if (!CommonUtil.isBlank(distroName)) {
         for (HadoopStack hadoopStack : hadoopStacks) {
            if (distroName.equalsIgnoreCase(hadoopStack.getDistro())) {
               return hadoopStack;
            }
         }
         throw BddException.NOT_FOUND("Distro", distroName);
      } else {
         return softwareManager.getDefaultStack();
      }
   }

   private void updateInfrastructure(ClusterCreate cluster,
         SoftwareManager softwareManager, ClusterBlueprint blueprint) {
      softwareManager.updateInfrastructure(blueprint);
      applyInfraChanges(cluster, blueprint);
   }

   private LinkedHashMap<NetTrafficType, List<ClusterNetConfigInfo>> validateAndConvertNetNamesToNetConfigs(
         Map<NetTrafficType, List<String>> netNamesInfo, boolean isMaprDistro) {
      LinkedHashMap<NetTrafficType, List<ClusterNetConfigInfo>> netConfigs =
            new LinkedHashMap<NetTrafficType, List<ClusterNetConfigInfo>>();
      LinkedHashMap<String, Set<String>> port2names = new LinkedHashMap<String, Set<String>>();

      for (NetTrafficType type : netNamesInfo.keySet()) {
         netConfigs.put(type, new ArrayList<ClusterNetConfigInfo>());
         for (String name : netNamesInfo.get(type)) {
            NetworkEntity networkEntity = networkMgr.getNetworkEntityByName(name);

            String pg = networkEntity.getPortGroup();
            Boolean isGenerateHostname = networkEntity.getIsGenerateHostname();
            String hostnamePrefix = HostnameManager.getHostnamePrefix();

            ClusterNetConfigInfo netConfig = new ClusterNetConfigInfo(type, name, pg, networkEntity.getDnsType(), isGenerateHostname, hostnamePrefix);
            netConfigs.get(type).add(netConfig);

            if (!port2names.containsKey(pg)) {
               port2names.put(pg, new HashSet<String>());
            }
            port2names.get(pg).add(name);
         }
      }

      if (isMaprDistro && port2names.size() > 1) {
         throw BddException.MULTI_NETWORKS_FOR_MAPR_DISTRO();
      }

      // if nw1,nw2 are both refer to pg1, should not use them in one cluster
      for (String pg : port2names.keySet()) {
         if (port2names.get(pg).size() > 1) {
            throw BddException.PG_REFERENCED_MULTI_TIMES();
         }
      }

      return netConfigs;
   }

   private Map<NetTrafficType, List<String>> convertNetConfigsToNetNames(
         Map<NetTrafficType, List<ClusterNetConfigInfo>> netConfigs) {
      Map<NetTrafficType, List<String>> netNamesInfo =
            new HashMap<NetTrafficType, List<String>>();
      for (NetTrafficType type : netConfigs.keySet()) {
         netNamesInfo.put(type, new ArrayList<String>());
         for (ClusterNetConfigInfo config : netConfigs.get(type)) {
            netNamesInfo.get(type).add(config.getNetworkName());
         }
      }
      return netNamesInfo;
   }

   private void validateMemorySize(Set<NodeGroupEntity> nodeGroups,
         List<String> failedMsgList) {
      boolean validated = true;
      StringBuilder invalidNodeGroupNames = new StringBuilder();
      for (NodeGroupEntity nodeGroup : nodeGroups) {
         if (nodeGroup.getMemorySize() < Constants.MIN_MEM_SIZE) {
            validated = false;
            invalidNodeGroupNames.append(nodeGroup.getName()).append(",");
         }
      }
      if (!validated) {
         StringBuilder errorMsgBuff = new StringBuilder();
         invalidNodeGroupNames.delete(invalidNodeGroupNames.length() - 1,
               invalidNodeGroupNames.length());
         failedMsgList.add(errorMsgBuff
               .append(
                     "'memCapacityMB' cannot be less than "
                           + Constants.MIN_MEM_SIZE + " in group ")
               .append(invalidNodeGroupNames.toString())
               .append(" in order for nodes to run normally").toString());
      }
   }

   private void updateVhmJobTrackerPort(ClusterCreate cluster,
         ClusterEntity clusterEntity) {
      if (cluster.getConfiguration().containsKey("hadoop")) {
         @SuppressWarnings("unchecked")
         Map<String, Object> hadoopConfig =
               (Map<String, Object>) cluster.getConfiguration().get("hadoop");
         if (hadoopConfig.containsKey("mapred-site.xml")) {
            @SuppressWarnings("unchecked")
            Map<String, Object> maprConfig =
                  (Map<String, Object>) hadoopConfig.get("mapred-site.xml");
            String jobtrackerAddress =
                  (String) maprConfig.get("mapred.job.tracker.http.address");
            if (jobtrackerAddress != null) {
               String[] items = jobtrackerAddress.split(":");
               String port = items[items.length - 1];
               Pattern pattern = Pattern.compile("[0-9]{1,5}");
               Matcher matcher = pattern.matcher(port);
               if (matcher.matches()) {
                  clusterEntity.setVhmJobTrackerPort(port);
               }
            }
         }
      }
   }

   private boolean validateRacksInfo(ClusterCreate cluster,
         List<String> failedMsgList) {
      boolean valid = true;
      Map<String, NodeGroupCreate> allGroups =
            new TreeMap<String, NodeGroupCreate>();
      if (cluster.getNodeGroups() == null) {
         return valid;
      }

      List<RackInfo> racksInfo = rackInfoMgr.exportRackInfo();

      if ((cluster.getTopologyPolicy() == TopologyType.HVE || cluster
            .getTopologyPolicy() == TopologyType.RACK_AS_RACK)
            && racksInfo.isEmpty()) {
         valid = false;
         throw ClusterConfigException
               .TOPOLOGY_WITH_NO_MAPPING_INFO_EXIST(cluster.getTopologyPolicy()
                     .toString());
      }

      for (NodeGroupCreate nodeGroupCreate : cluster.getNodeGroups()) {
         allGroups.put(nodeGroupCreate.getName(), nodeGroupCreate);
      }

      for (NodeGroupCreate ngc : cluster.getNodeGroups()) {
         PlacementPolicy policies = ngc.getPlacementPolicies();
         if (policies != null && policies.getGroupAssociations() != null) {
            continue;
         }

         if (ngc.getStorage() != null
               && ngc.getStorage().getType() != null
               && ngc.getStorage().getType()
                     .equals(DatastoreType.SHARED.toString())) {
            continue;
         }

         if (policies != null && policies.getGroupRacks() != null) {
            if (racksInfo.isEmpty()) {
               valid = false;
               throw ClusterConfigException
                     .RACKPOLICY_WITH_NO_MAPPING_INFO_EXIST(ngc.getName());
            }

            GroupRacks r = policies.getGroupRacks();
            GroupRacksType rackType = r.getType();
            Set<String> specifiedRacks =
                  new HashSet<String>(Arrays.asList(r.getRacks()));

            if (rackType.equals(GroupRacksType.SAMERACK)
                  && specifiedRacks.size() != 1) {
               throw ClusterConfigException.MUST_DEFINE_ONE_RACK(ngc.getName());
            }

            List<String> intersecRacks = new ArrayList<String>();
            int intersecHostNum = 0;
            int maxIntersecHostNum = 0;

            for (RackInfo rackInfo : racksInfo) {
               if (specifiedRacks.isEmpty() || specifiedRacks.size() == 0
                     || specifiedRacks.contains(rackInfo.getName())) {
                  intersecHostNum += rackInfo.getHosts().size();
                  intersecRacks.add(rackInfo.getName());
                  if (rackInfo.getHosts().size() > maxIntersecHostNum) {
                     maxIntersecHostNum = rackInfo.getHosts().size();
                  }
               }
            }

            if (intersecRacks.size() == 0) {
               valid = false;
               throw ClusterConfigException.NO_VALID_RACK(ngc.getName());
            }

            if (ngc.calculateHostNum() != null) {
               if (rackType.equals(GroupRacksType.ROUNDROBIN)
                     && ngc.calculateHostNum() > intersecHostNum) {
                  valid = false;
                  throw ClusterConfigException.LACK_PHYSICAL_HOSTS(
                        ngc.calculateHostNum(), ngc.getName(), intersecHostNum);
               } else if (rackType.equals(GroupRacksType.SAMERACK)
                     && ngc.calculateHostNum() > maxIntersecHostNum) {
                  valid = false;
                  throw ClusterConfigException.LACK_PHYSICAL_HOSTS(
                        ngc.calculateHostNum(), ngc.getName(),
                        maxIntersecHostNum);
               }
            }

            if (specifiedRacks.isEmpty()) {
               r.setRacks(new String[0]);
            } else {
               r.setRacks(intersecRacks.toArray(new String[intersecRacks.size()]));
            }
         }
      }
      return valid;
   }

   private Set<NodeGroupEntity> convertNodeGroupsToEntities(Gson gson,
         ClusterEntity clusterEntity, String distro, NodeGroupCreate[] groups,
         boolean validateWhiteList) {
      Set<NodeGroupEntity> nodeGroups = new LinkedHashSet<NodeGroupEntity>();
      for (NodeGroupCreate group : groups) {
         NodeGroupEntity groupEntity =
               convertGroup(gson, clusterEntity, group, distro,
                     validateWhiteList);
         if (groupEntity != null) {
            nodeGroups.add(groupEntity);
         }
      }
      return nodeGroups;
   }

   public NodeGroupEntity convertGroup(Gson gson, ClusterEntity clusterEntity,
         NodeGroupCreate group, String distro, boolean validateWhiteList) {
      NodeGroupEntity groupEntity = new NodeGroupEntity();
      if (group.getRoles() == null || group.getRoles().isEmpty()) {
         throw ClusterConfigException.NO_HADOOP_ROLE_SPECIFIED(group.getName());
      }

      groupEntity.setCluster(clusterEntity);
      String templateId = this.nodeTemplateService.getNodeTemplateNameByMoid(clusterEntity.getTemplateId());
      int cpuNum = group.getCpuNum() == null ? 0 : group.getCpuNum();
      if (!VcVmUtil.validateCPU(clusterEntity.getTemplateId(), cpuNum)) {
         throw VcProviderException.CPU_NUM_NOT_MULTIPLE_OF_CORES_PER_SOCKET(group.getName(), templateId);
      }

      groupEntity.setCpuNum(cpuNum);
      groupEntity.setDefineInstanceNum(group.getInstanceNum());
      groupEntity.setMemorySize(group.getMemCapacityMB() == null ? 0 : group
            .getMemCapacityMB());
      groupEntity.setSwapRatio(group.getSwapRatio());
      groupEntity.setName(group.getName());
      groupEntity.setNodeType(group.getInstanceType());
      groupEntity.setReservedCpu_ratio(group.getReservedCpuRatio());
      groupEntity.setReservedMem_ratio(group.getReservedMemRatio());

      PlacementPolicy policies = group.getPlacementPolicies();
      if (policies != null) {
         List<GroupAssociation> associons = policies.getGroupAssociations();
         if (associons != null) {
            Set<NodeGroupAssociation> associonEntities =
                  new HashSet<NodeGroupAssociation>();
            for (GroupAssociation a : associons) {
               NodeGroupAssociation ae = new NodeGroupAssociation();
               ae.setAssociationType(a.getType());
               ae.setNodeGroup(groupEntity);
               ae.setReferencedGroup(a.getReference());
               associonEntities.add(ae);
            }
            groupEntity.setGroupAssociations(associonEntities);
         }
         if (policies.getInstancePerHost() != null) {
            groupEntity.setInstancePerHost(policies.getInstancePerHost());
         }

         if (policies.getGroupRacks() != null) {
            groupEntity.setGroupRacks((new Gson()).toJson(policies
                  .getGroupRacks()));
         }
      }

      if (group.getRpNames() != null && group.getRpNames().size() > 0) {
         groupEntity.setVcRpNameList(group.getRpNames());
      }

      /*
       * do not support node group level networks temporarilly
      if (group.getNetworkNames() != null && group.getNetworkNames().size() > 0) {
         groupEntity.setVcNetworkNames(group.getNetworkNames());
      }
      */

      Set<String> roles = new LinkedHashSet<String>();
      convertStorage(group, groupEntity, roles);
      roles.addAll(group.getRoles());

      groupEntity.setRoles(gson.toJson(roles));

      if (group.getInstanceNum() <= 0) {
         logger.warn("Zero or negative instance number for group "
               + group.getName() + ", remove the group from cluster spec.");
         return null;
      }

      List<String> dsNames = groupEntity.getVcDatastoreNameList();
      if (dsNames == null) {
         dsNames = clusterEntity.getVcDatastoreNameList();
      }
      Set<String> sharedPattern;
      Set<String> localPattern;
      if (dsNames != null) {
         sharedPattern = datastoreMgr.getSharedDatastoresByNames(dsNames);
         localPattern = datastoreMgr.getLocalDatastoresByNames(dsNames);
      } else {
         sharedPattern = datastoreMgr.getAllSharedDatastores();
         localPattern = datastoreMgr.getAllLocalDatastores();
      }

      SoftwareManager softwareManager =
            getSoftwareManager(clusterEntity.getAppManager());
      CommonClusterExpandPolicy.expandGroupInstanceType(groupEntity, group,
            sharedPattern, localPattern, softwareManager);
      String haFlag = group.getHaFlag();
      if (haFlag == null) {
         groupEntity.setHaFlag(Constants.HA_FLAG_OFF);
      } else {
         groupEntity.setHaFlag(haFlag);
      }
      LatencyPriority latencySensitivity = group.getLatencySensitivity();
      if(latencySensitivity != null)
         groupEntity.setLatencySensitivity(latencySensitivity);
      else
         groupEntity.setLatencySensitivity(LatencyPriority.NORMAL);
      groupEntity.setReservedCpu_ratio(group.getReservedCpuRatio());
      groupEntity.setReservedMem_ratio(group.getReservedMemRatio());
      if (group.getConfiguration() != null
            && group.getConfiguration().size() > 0) {
         groupEntity.setHadoopConfig(gson.toJson(group.getConfiguration()));
      }
      // set vm folder path
      groupEntity.setVmFolderPath(clusterEntity);
      logger.debug("finished to convert node group config for "
            + group.getName());
      return groupEntity;
   }

   private void convertStorage(NodeGroupCreate group,
         NodeGroupEntity groupEntity, Set<String> roles) {
      if (group.getStorage() != null) {
         groupEntity.setStorageSize(group.getStorage().getSizeGB());
         groupEntity.setDiskNum(getDiskNumberForNodeGroup(group));
         groupEntity.setShareDatastore(isShareDatastoreForNodeGroup(group));
         //currently, ignore input from CLI and hard code here
         String storageType = group.getStorage().getType();
         if (storageType != null) {
            if (storageType.equalsIgnoreCase(DatastoreType.TEMPFS.name())) {
               groupEntity.setStorageType(DatastoreType.TEMPFS);
            } else if (storageType.equalsIgnoreCase(DatastoreType.LOCAL.name())) {
               groupEntity.setStorageType(DatastoreType.LOCAL);
            } else {
               groupEntity.setStorageType(DatastoreType.SHARED);
            }
         }
         groupEntity.setVcDatastoreNameList(group.getStorage().getDsNames());
         groupEntity.setSdDatastoreNameList(group.getStorage()
               .getDsNames4System());
         groupEntity.setDdDatastoreNameList(group.getStorage()
               .getDsNames4Data());
      }

      if (groupEntity.getStorageType() == DatastoreType.LOCAL) {
         // only when explicitly set to local, we'll choose local storage
         if (group.getHaFlag() != null
               && Constants.HA_FLAG_FT.equals(group.getHaFlag().toLowerCase())) {
            throw ClusterConfigException.LOCAL_STORAGE_USED_FOR_FT_GROUP(group
                  .getName());
         }
      }
   }

   @Transactional(readOnly = true)
   public ClusterCreate getClusterConfig(String clusterName) {
      return getClusterConfig(clusterName, true);
   }

   @Transactional(readOnly = true)
   public ClusterCreate getClusterConfig(String clusterName, boolean needAllocIp) {
      ClusterEntity clusterEntity = clusterEntityMgr.findByName(clusterName);
      if (clusterEntity == null) {
         throw ClusterConfigException.CLUSTER_CONFIG_NOT_FOUND(clusterName);
      }
      ClusterCreate clusterConfig = new ClusterCreate();
      clusterConfig.setName(clusterEntity.getName());
      clusterConfig.setAppManager(clusterEntity.getAppManager());
      clusterConfig.setDistro(clusterEntity.getDistro());
      Map<NetTrafficType, List<ClusterNetConfigInfo>> networkConfigInfo = clusterEntity.getNetworkConfigInfo();
      if (networkConfigInfo != null && !networkConfigInfo.isEmpty()) {
         for (NetTrafficType trafficType : networkConfigInfo.keySet()) {
            clusterConfig.setHostnamePrefix(networkConfigInfo.get(trafficType).get(0).getHostnamePrefix());
            break;
         }
      }
      convertClusterConfig(clusterEntity, clusterConfig, needAllocIp);

      Gson gson =
            new GsonBuilder().excludeFieldsWithoutExposeAnnotation().create();
      String manifest = gson.toJson(clusterConfig);
      logger.debug("final cluster spec: " + manifest);
      return clusterConfig;
   }

   @SuppressWarnings("unchecked")
   private void convertClusterConfig(ClusterEntity clusterEntity,
         ClusterCreate clusterConfig, boolean needAllocIp) {
      logger.debug("begin to expand config for cluster "
            + clusterEntity.getName());
      clusterConfig.setDistroVendor(clusterEntity.getDistroVendor());
      clusterConfig.setDistroVersion(clusterEntity.getDistroVersion());
      clusterConfig.setAppManager(clusterEntity.getAppManager());
      clusterConfig.setHttpProxy(httpProxy);
      clusterConfig.setNoProxy(noProxy);
      clusterConfig.setTopologyPolicy(clusterEntity.getTopologyPolicy());
      clusterConfig.setPassword(clusterEntity.getPassword());
      clusterConfig.setTemplateName(this.nodeTemplateService.getNodeTemplateNameByMoid(clusterEntity.getTemplateId()));

      Map<String, String> hostToRackMap = rackInfoMgr.exportHostRackMap();
      if ((clusterConfig.getTopologyPolicy() == TopologyType.RACK_AS_RACK || clusterConfig
            .getTopologyPolicy() == TopologyType.HVE)
            && hostToRackMap.isEmpty()) {
         logger.error("trying to use host-rack topology which is absent");
         throw ClusterConfigException.INVALID_TOPOLOGY_POLICY(
               clusterConfig.getTopologyPolicy(), "no rack information.");
      }
      clusterConfig.setHostToRackMap(hostToRackMap);

      if (clusterEntity.getVcRpNames() != null) {
         logger.debug("resource pool specified at cluster level.");
         String[] rpNames =
               clusterEntity.getVcRpNameList().toArray(
                     new String[clusterEntity.getVcRpNameList().size()]);
         List<VcCluster> vcClusters =
               rpMgr.getVcResourcePoolByNameList(rpNames);
         clusterConfig.setVcClusters(vcClusters);
         clusterConfig.setRpNames(clusterEntity.getVcRpNameList());
      } else {
         // set all vc clusters
         clusterConfig.setVcClusters(rpMgr.getAllVcResourcePool());
         logger.debug("no resource pool config at cluster level.");
      }

      if (clusterEntity.getVcDatastoreNameList() != null) {
         logger.debug("datastore specified at cluster level.");
         Set<String> sharedPattern =
               datastoreMgr.getSharedDatastoresByNames(clusterEntity
                     .getVcDatastoreNameList());
         clusterConfig.setSharedDatastorePattern(sharedPattern);
         Set<String> localPattern =
               datastoreMgr.getLocalDatastoresByNames(clusterEntity
                     .getVcDatastoreNameList());
         clusterConfig.setLocalDatastorePattern(localPattern);
         clusterConfig.setDsNames(clusterEntity.getVcDatastoreNameList());
      } else {
         // set all shared and local datastores
         clusterConfig.setSharedDatastorePattern(datastoreMgr
               .getAllSharedDatastores());
         clusterConfig.setLocalDatastorePattern(datastoreMgr
               .getAllLocalDatastores());
         logger.debug("no datastore config at cluster level.");
      }
      List<NodeGroupCreate> nodeGroups = new ArrayList<NodeGroupCreate>();

      // TODO need more role checks

      Set<NodeGroupEntity> nodeGroupEntities = clusterEntity.getNodeGroups();
      long instanceNum = 0;

      for (NodeGroupEntity ngEntity : nodeGroupEntities) {
         NodeGroupCreate group =
               convertNodeGroups(clusterEntity, ngEntity,
                     clusterEntity.getName());
         nodeGroups.add(group);
         instanceNum += group.getInstanceNum();
      }
      clusterConfig.setNodeGroups(nodeGroups
            .toArray(new NodeGroupCreate[nodeGroups.size()]));

      List<String> networkNames = clusterEntity.fetchNetworkNameList();

      // TODO: refactor this function to support nodeGroup level networks
      List<NetworkAdd> networkingAdds =
            allocatNetworkIp(networkNames, clusterEntity, instanceNum,
                  needAllocIp);
      clusterConfig.setNetworkings(networkingAdds);
      clusterConfig.setNetworkConfig(convertNetConfigsToNetNames(clusterEntity
            .getNetworkConfigInfo()));

      if (clusterEntity.getHadoopConfig() != null) {
         Map<String, Object> hadoopConfig =
               (new Gson())
                     .fromJson(clusterEntity.getHadoopConfig(), Map.class);
         clusterConfig.setConfiguration(hadoopConfig);
      }
      if (!CommonUtil.isBlank(clusterEntity.getAdvancedProperties())) {
         Gson gson = new Gson();
         Map<String, String> advancedProperties =
               gson.fromJson(clusterEntity.getAdvancedProperties(), Map.class);
         clusterConfig.setExternalHDFS(advancedProperties.get("ExternalHDFS"));
         clusterConfig.setExternalMapReduce(advancedProperties
               .get("ExternalMapReduce"));
         clusterConfig.setLocalRepoURL(advancedProperties.get("LocalRepoURL"));
         clusterConfig.setClusterCloneType(advancedProperties.get("ClusterCloneType"));
         clusterConfig.setExternalNamenode(advancedProperties.get("ExternalNamenode"));
         clusterConfig.setExternalSecondaryNamenode(advancedProperties.get("ExternalSecondaryNamenode"));
         if (advancedProperties.get("ExternalDatanodes") != null) {
            clusterConfig.setExternalDatanodes(gson.fromJson(gson.toJson(advancedProperties.get("ExternalDatanodes")), HashSet.class));
         }
      }

      // for clusters from previous releases, the clone type is fast clone
      setDefaultClusterCloneType(clusterConfig);

      if(!CommonUtil.isBlank(clusterEntity.getInfraConfig())) {
         clusterConfig.setInfrastructure_config(InfrastructureConfigUtils.read(clusterEntity.getInfraConfig()));
      }
   }

   private List<NetworkAdd> allocatNetworkIp(List<String> networkNames,
         ClusterEntity clusterEntity, long instanceNum, boolean needAllocIp) {
      List<NetworkAdd> networkings = new ArrayList<NetworkAdd>();

      for (String networkName : networkNames) {
         NetworkEntity networkEntity =
               networkMgr.getNetworkEntityByName(networkName);

         if (needAllocIp) {
            NetworkAdd network = new NetworkAdd();
            network.setPortGroup(networkEntity.getPortGroup());
            network.setName(networkName);
            network
                  .setDhcp(networkEntity.getAllocType() == NetworkEntity.AllocType.DHCP);
            if (!network.getIsDhcp()) {
               logger.debug("using static ip.");
               List<IpBlockEntity> ipBlockEntities =
                     networkMgr.getAllocatedIpBlocks(networkEntity,
                           clusterEntity.getId());
               long allocatedIpNum = IpBlockEntity.count(ipBlockEntities);
               if (allocatedIpNum < instanceNum) {
                  long newNum = instanceNum - allocatedIpNum;
                  List<IpBlockEntity> newIpBlockEntities =
                        networkMgr.alloc(networkEntity, clusterEntity.getId(),
                              newNum);
                  ipBlockEntities.addAll(newIpBlockEntities);
               }
               network.setDns1(networkEntity.getDns1());
               network.setDns2(networkEntity.getDns2());
               network.setGateway(networkEntity.getGateway());
               network.setNetmask(networkEntity.getNetmask());
               List<IpBlock> ips = new ArrayList<IpBlock>();
               for (IpBlockEntity ipBlockEntity : ipBlockEntities) {
                  IpBlock ip = new IpBlock();
                  ip.setBeginIp(ipBlockEntity.getBeginAddress());
                  ip.setEndIp(ipBlockEntity.getEndAddress());
                  ips.add(ip);
               }
               network.setIpBlocks(ips);
            }
            networkings.add(network);
         }
      }
      return networkings;
   }

   @SuppressWarnings("unchecked")
   private NodeGroupCreate convertNodeGroups(ClusterEntity clusterEntity,
         NodeGroupEntity ngEntity, String clusterName) {
      Gson gson = new Gson();
      List<String> groupRoles = gson.fromJson(ngEntity.getRoles(), List.class);
      NodeGroupCreate group = new NodeGroupCreate();
      group.setName(ngEntity.getName());
      group.setRoles(groupRoles);
      int cpu = ngEntity.getCpuNum();
      if (cpu > 0) {
         group.setCpuNum(cpu);
      }

      int memory = ngEntity.getMemorySize();
      if (memory > 0) {
         group.setMemCapacityMB(memory);
      }

      Float swapRatio = ngEntity.getSwapRatio();
      if (swapRatio != null && swapRatio >= 0) {
         group.setSwapRatio(swapRatio);
      }

      if (ngEntity.getNodeType() != null) {
         group.setInstanceType(ngEntity.getNodeType());
      }

      group.setInstanceNum(ngEntity.getDefineInstanceNum());

      Integer instancePerHost = ngEntity.getInstancePerHost();
      Set<NodeGroupAssociation> associonEntities =
            ngEntity.getGroupAssociations();
      String ngRacks = ngEntity.getGroupRacks();
      if (instancePerHost == null
            && (associonEntities == null || associonEntities.isEmpty())
            && ngRacks == null) {
         group.setPlacementPolicies(null);
      } else {
         PlacementPolicy policies = new PlacementPolicy();
         policies.setInstancePerHost(instancePerHost);
         if (ngRacks != null) {
            policies.setGroupRacks(new Gson().fromJson(ngRacks,
                  GroupRacks.class));
         }
         if (associonEntities != null) {
            List<GroupAssociation> associons =
                  new ArrayList<GroupAssociation>(associonEntities.size());
            for (NodeGroupAssociation ae : associonEntities) {
               GroupAssociation a = new GroupAssociation();
               a.setReference(ae.getReferencedGroup());
               a.setType(ae.getAssociationType());
               associons.add(a);
            }
            policies.setGroupAssociations(associons);
         }

         group.setPlacementPolicies(policies);
      }

      String rps = ngEntity.getVcRpNames();
      if (rps != null && rps.length() > 0) {
         logger.debug("resource pool specified at node group "
               + ngEntity.getName());
         String[] rpNames = gson.fromJson(rps, String[].class);
         List<VcCluster> vcClusters =
               rpMgr.getVcResourcePoolByNameList(rpNames);
         group.setVcClusters(vcClusters);
         group.setRpNames(Arrays.asList(rpNames));
      }

      expandGroupStorage(ngEntity, group);
      group.setHaFlag(ngEntity.getHaFlag());
      group.setLatencySensitivity(ngEntity.getLatencySensitivity());
      group.setReservedCpuRatio(ngEntity.getReservedCpu_ratio());
      group.setReservedMemRatio(ngEntity.getReservedMem_ratio());
      if (ngEntity.getHadoopConfig() != null) {
         Map<String, Object> hadoopConfig =
               (new Gson()).fromJson(ngEntity.getHadoopConfig(), Map.class);
         group.setConfiguration(hadoopConfig);
      }

      group.setVmFolderPath(ngEntity.getVmFolderPath());
      return group;
   }

   private void expandGroupStorage(NodeGroupEntity ngEntity,
         NodeGroupCreate group) {
      int storageSize = ngEntity.getStorageSize();
      DatastoreType storageType = ngEntity.getStorageType();

      List<String> storeNames = ngEntity.getVcDatastoreNameList();
      List<String> dataDiskStoreNames = ngEntity.getDdDatastoreNameList();
      List<String> systemDiskStoreNames = ngEntity.getSdDatastoreNameList();

      logger.debug("expanding group storage for cluster " + ngEntity.getCluster().getName());
      if (storageSize <= 0 && storageType == null
            && (storeNames == null || storeNames.isEmpty())) {
         logger.debug("no storage specified for node group "
               + ngEntity.getName());
      }

      logger.debug("storage size is " + storageSize + " for node group "
            + ngEntity.getName());
      logger.debug("storage type is " + storageType + " for node group "
            + ngEntity.getName());
      logger.debug("diskNum is " + ngEntity.getDiskNum() + " for node group "
            + ngEntity.getName());
      logger.debug("shareDatastore is " + ngEntity.isShareDatastore() + " for node group "
            + ngEntity.getName());
      logger.debug("storage name pattern is " + storeNames + " for node group "
            + ngEntity.getName());
      logger.debug("system disk storage name pattern is "
            + systemDiskStoreNames + " for node group " + ngEntity.getName());
      logger.debug("data disk storage name pattern is " + dataDiskStoreNames
            + " for node group " + ngEntity.getName());
      StorageRead storage = new StorageRead();
      group.setStorage(storage);
      storage.setSizeGB(storageSize);
      storage.setDiskNum(ngEntity.getDiskNum());
      storage.setShareDatastore(ngEntity.isShareDatastore());
      if (storageType != null) {
         storage.setType(storageType.toString().toLowerCase());
      }

      if (systemDiskStoreNames != null && !systemDiskStoreNames.isEmpty()) {
         // when getDatastoreNamePattern return null, the ImagestoreNamePattern will be set to proper
         // values in NodeGroupCreate.getImagestoreNamePattern()
         storage.setImagestoreNamePattern(getDatastoreNamePattern(systemDiskStoreNames));
         storage.setDsNames4System(systemDiskStoreNames);
      } else {
         storage.setImagestoreNamePattern(getDatastoreNamePattern(storeNames));
      }

      if (dataDiskStoreNames != null && !dataDiskStoreNames.isEmpty()) {
         // when getDatastoreNamePattern return null, the DiskstoreNamePattern will be set to proper
         // values in NodeGroupCreate.getDiskstoreNamePattern()
         storage.setDiskstoreNamePattern(getDatastoreNamePattern(dataDiskStoreNames));
         storage.setDsNames4Data(dataDiskStoreNames);
      } else {
         storage.setDiskstoreNamePattern(getDatastoreNamePattern(storeNames));
      }
      storage.setShares(ngEntity.getCluster().getIoShares());

      // set storage split policy based on group roles
      SoftwareManager softwareManager =
            getSoftwareManager(ngEntity.getCluster().getAppManager());
      if (softwareManager.twoDataDisksRequired(group.toNodeGroupInfo())) {
         logger.debug("use bi_sector disk layout for zookeeper only group.");
         storage.setSplitPolicy(DiskSplitPolicy.BI_SECTOR);
      } else {
         if (storage.getType().equalsIgnoreCase(DatastoreType.LOCAL.toString())) {
            logger.debug("use even split disk layout for local datastore.");
            storage.setSplitPolicy(DiskSplitPolicy.EVEN_SPLIT);
         } else {
            logger.debug("use aggregate split disk layout.");
            storage.setSplitPolicy(DiskSplitPolicy.AGGREGATE);
         }
      }

      // set disk scsi controller type
      setDiskAttributes(storageType, storage, storeNames);
   }

   private void setDiskAttributes(DatastoreType storageType,
         StorageRead storage, List<String> storeNames) {
      if (storageType == null) {
         // check store names to see if local type storage is chosen.
         Set<String> storePattern =
               datastoreMgr.getLocalDatastoresByNames(storeNames);
         if (storePattern != null && !storePattern.isEmpty()) {
            logger.info("datastore type is not set, but local datastore is used. Set scsi controller type to paravirtual");
            storage
                  .setControllerType(DiskScsiControllerType.PARA_VIRTUAL_CONTROLLER);
            storage.setAllocType(AllocationType.THICK.name());
         } else {
            storage.setControllerType(DiskScsiControllerType.LSI_CONTROLLER);
            storage.setAllocType(AllocationType.THIN.name());
         }
         return;
      }

      DiskScsiControllerType systemSwapDiskControllerType = CommonUtil.getSystemAndSwapControllerType();
      DiskScsiControllerType dataDiskControllerType = CommonUtil.getDataDiskControllerType();
      if ( systemSwapDiskControllerType == DiskScsiControllerType.PARA_VIRTUAL_CONTROLLER
            && dataDiskControllerType == DiskScsiControllerType.LSI_CONTROLLER ) {
         logger.warn("The data disk controller type cannot be set to LSI_CONTROLLER when "
               + "the system and swap disk controller type is PARA_VIRTUAL_CONTROLLER."
               + "The PARA_VIRTUAL_CONTROLLER will be used for data disk.");
         dataDiskControllerType = DiskScsiControllerType.PARA_VIRTUAL_CONTROLLER;
      }
      storage.setControllerType(dataDiskControllerType);
      storage.setAllocType(AllocationType.THICK.name());
      if ( storageType == DatastoreType.SHARED ) {
         storage.setAllocType(AllocationType.THIN.name());
      }
   }

   private List<String> getDatastoreNamePattern(List<String> storeNames) {
      if (storeNames == null || storeNames.isEmpty()) {
         return null;
      }
      Set<String> storePattern = null;
      storePattern = datastoreMgr.getDatastoresByNames(storeNames);
      if (storePattern == null || storePattern.isEmpty()) {
         String datastoreNames = new Gson().toJson(storeNames);
         logger.error("datastore " + datastoreNames + " not found in BDE");
         throw BddException.NOT_FOUND("datastore", datastoreNames);
      }
      return new ArrayList<>(storePattern);
   }

   @Transactional
   public void updateAppConfig(String clusterName, ClusterCreate clusterCreate) {
      logger.debug("Update configuration for cluster " + clusterName);

      ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);

      if (cluster == null) {
         logger.error("cluster " + clusterName + " does not exist");
         throw BddException.NOT_FOUND("Cluster", clusterName);
      }
      SoftwareManager softwareManager =
            getSoftwareManager(cluster.getAppManager());
      // read distro and distroVersion from ClusterEntity and set to ClusterCreate
      clusterCreate.setDistro(cluster.getDistro());
      clusterCreate.setDistroVersion(cluster.getDistroVersion());
      if (!CommonUtil.isBlank(cluster.getAdvancedProperties())) {
         Gson gson = new Gson();
         Map<String, String> advancedProperties =
               gson.fromJson(cluster.getAdvancedProperties(), Map.class);
         clusterCreate.setExternalHDFS(advancedProperties.get("ExternalHDFS"));
         clusterCreate.setExternalMapReduce(advancedProperties
               .get("ExternalMapReduce"));
         clusterCreate.setLocalRepoURL(advancedProperties.get("LocalRepoURL"));
         clusterCreate.setClusterCloneType(advancedProperties.get("ClusterCloneType"));
      }

      // for clusters from previous releases, the clone type is fast clone
      setDefaultClusterCloneType(clusterCreate);

      // only check roles validity in server side, but not in CLI and GUI, because roles info exist in server side.
      ClusterBlueprint blueprint = clusterCreate.toBlueprint();
      try {
         softwareManager.validateBlueprint(blueprint);
      } catch (ValidationException e) {
         throw ClusterConfigException.INVALID_SPEC(e.getFailedMsgList());
      }

      updateInfrastructure(clusterCreate, softwareManager, blueprint);
      Map<String, Object> clusterLevelConfig = clusterCreate.getConfiguration();

      if (clusterLevelConfig != null && clusterLevelConfig.size() > 0) {
         logger.debug("Cluster level app config is updated.");
         cluster.setHadoopConfig((new Gson()).toJson(clusterLevelConfig));
         updateVhmJobTrackerPort(clusterCreate, cluster);
      } else {
         logger.debug("cluster configuration is not set in cluster spec, so treat it as an empty configuration.");
         cluster.setHadoopConfig(null);
      }
      setAdvancedProperties(clusterCreate.getExternalHDFS(),
            clusterCreate.getExternalMapReduce(),
            clusterCreate.getLocalRepoURL(),
            clusterCreate.getExternalNamenode(),
            clusterCreate.getExternalSecondaryNamenode(),
            clusterCreate.getExternalDatanodes(),
            clusterCreate.getClusterCloneType(),
            cluster);
      updateNodegroupAppConfig(clusterCreate, cluster,
            clusterCreate.isValidateConfig());
   }

   private void updateNodegroupAppConfig(ClusterCreate clusterCreate,
         ClusterEntity cluster, boolean validateWhiteList) {
      Gson gson = new Gson();
      Set<NodeGroupEntity> groupEntities = cluster.getNodeGroups();
      Map<String, NodeGroupEntity> groupMap =
            new HashMap<String, NodeGroupEntity>();
      for (NodeGroupEntity entity : groupEntities) {
         groupMap.put(entity.getName(), entity);
      }

      Set<String> updatedGroups = new HashSet<String>();
      NodeGroupCreate[] groupCreates = clusterCreate.getNodeGroups();
      if (groupCreates == null) {
         return;
      }
      for (NodeGroupCreate groupCreate : groupCreates) {
         Map<String, Object> groupConfig = groupCreate.getConfiguration();
         if (groupConfig != null && groupConfig.size() > 0) {
            NodeGroupEntity groupEntity = groupMap.get(groupCreate.getName());
            // validate hadoop config
            groupEntity.setHadoopConfig(gson.toJson(groupConfig));
            updatedGroups.add(groupCreate.getName());
         }
      }
      for (NodeGroupEntity entity : groupEntities) {
         if (updatedGroups.contains(entity.getName())) {
            continue;
         }
         entity.setHadoopConfig(null);
      }
   }

   /**
    * validate if rack topology of all hosts is uploaded
    *
    * @param hosts
    * @param topology
    */
   @Transactional
   public void validateRackTopologyUploaded(Set<String> hosts, String topology) {
      Map<String, String> rackMap = rackInfoMgr.exportHostRackMap();
      List<String> invalidHosts = new ArrayList<String>();
      for (String hostName : hosts) {
         if (!rackMap.containsKey(hostName)) {
            invalidHosts.add(hostName);
         }
      }
      if (invalidHosts.size() > 0) {
         if (topology.equalsIgnoreCase(TopologyType.HVE.toString())
               || topology.equalsIgnoreCase(TopologyType.RACK_AS_RACK
                     .toString())) {
            throw ClusterConfigException
                  .TOPOLOGY_WITH_NO_MAPPING_INFO_EXIST(topology);
         }
      }
   }

   /**
    * build rack topology of the nodes according to the topology
    *
    * @param nodes
    * @param topology
    * @return
    */
   @Transactional
   public Map<String, String> buildTopology(List<NodeRead> nodes,
         String topology) {
      topology = topology.toUpperCase();
      Map<String, String> map = new HashMap<String, String>();
      Map<NetTrafficType, List<IpConfigInfo>> ipConfigMap = null;
      Map<String, String> hostRackMap = rackInfoMgr.exportHostRackMap();
      Iterator<Entry<NetTrafficType, List<IpConfigInfo>>> ipConfigsIt = null;
      Entry<NetTrafficType, List<IpConfigInfo>> ipConfigEntry = null;
      List<IpConfigInfo> ipConfigs = null;
      for (NodeRead node : nodes) {
         ipConfigMap = node.getIpConfigs();
         ipConfigsIt = ipConfigMap.entrySet().iterator();
         while (ipConfigsIt.hasNext()) {
            ipConfigEntry = ipConfigsIt.next();
            ipConfigs = ipConfigEntry.getValue();
            if (ipConfigs != null && ipConfigs.size() > 0) {
               for (IpConfigInfo ipConfig : ipConfigs) {
                  if (!CommonUtil.isBlank(ipConfig.getIpAddress())) {
                     switch (topology) {
                     case "HOST_AS_RACK":
                        map.put(ipConfig.getIpAddress(),
                              "/" + node.getHostName());
                        break;
                     case "RACK_AS_RACK":
                        map.put(ipConfig.getIpAddress(),
                              "/" + hostRackMap.get(node.getHostName()));
                        break;
                     case "HVE":
                        map.put(ipConfig.getIpAddress(),
                              "/" + hostRackMap.get(node.getHostName()) + "/"
                                    + node.getHostName());
                        break;
                     case "NONE":
                     default:
                        map.put(ipConfig.getIpAddress(), "/default-rack");
                        break;
                     }
                  }
               }
            }
         }
      }
      return map;
   }

   public SoftwareManager getSoftwareManager(String appManager) {
      return softwareManagerCollector.getSoftwareManager(appManager);
   }

   private boolean validateLocalRepoURL(String localRepoURL) {
      boolean succ = true;
      HttpClientBuilder builder = HttpClientBuilder.create();
      CloseableHttpClient httpClient = builder.build();

      // test the connection to the given url
      try {
         HttpGet httpGet = new HttpGet(localRepoURL);
         HttpResponse resp = httpClient.execute(httpGet);
         StatusLine status = resp.getStatusLine();
         if (status.getStatusCode() >= 400) {
            succ = false;
         }
      } catch (Exception e) {
         succ = false;
         logger.error(e.getMessage());
      } finally {
         if (null != httpClient) {
            try {
               httpClient.close();
            } catch (IOException e) {
               logger.error("Unknown errors in closing the http connection.");
            }
         }
      }

      return succ;
   }

   private void setDefaultClusterCloneType(ClusterCreate clusterCreate) {
      String cloneType = clusterCreate.getClusterCloneType();
      if (CommonUtil.isBlank(cloneType)) {
         clusterCreate.setClusterCloneType(Constants.CLUSTER_CLONE_TYPE_FAST_CLONE);
      }
   }

   /*
    * Get the disks number for each node group specified in cluster spec file or in global settings.
    * The total storage of a node splits into the specified number of disks.
    * 0 by default which means using the default storage split policy defined in PlacementPlanner#placeDisk().
    */
   private int getDiskNumberForNodeGroup(NodeGroupCreate group) {
      String storageType = null;
      Integer diskNumber = null;
      if (group.getStorage() != null) {
         storageType = group.getStorage().getType();
         diskNumber = group.getStorage().getDiskNum();
      }
      if (storageType == null) {
         storageType = DatastoreType.LOCAL.name();
      }
      if (diskNumber == null) {
         diskNumber = Configuration.getInt(String.format("storage.%1$s.disk_number_per_node", storageType.toLowerCase()), 0);
      }

      int maxDiskNum = ConfigInfo.getMaxDiskNumPerNode();
      logger.debug("maxDiskNum is " + maxDiskNum);
      if (diskNumber < 0) {
         diskNumber = 0;
      } else if (diskNumber > maxDiskNum) {
         throw ClusterConfigException.DISK_NUM_EXCEED_LIMIT(group.getName(), diskNumber, maxDiskNum);
      }


      return diskNumber;
   }

   /*
    * Get the shareDatastore setting for each node group specified in cluster spec file or in global settings.
    * true by default which means sharing a datastore with other disks rather than allocating the whole datastore to a single disk.
    */
   private boolean isShareDatastoreForNodeGroup(NodeGroupCreate group) {
      String storageType = null;
      Boolean isShareDatastore = null;
      if (group.getStorage() != null) {
         storageType = group.getStorage().getType();
         isShareDatastore = group.getStorage().isShareDatastore();
      }
      if (storageType == null) {
         storageType = DatastoreType.LOCAL.name();
      }
      if (isShareDatastore != null) {
         return isShareDatastore;
      } else {
         return Configuration.getBoolean(String.format("storage.%1$s.is_share_datastore", storageType.toLowerCase()), true);
      }
   }
}