ClusteringService.java

/***************************************************************************
 * Copyright (c) 2012-2015 VMware, Inc. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ***************************************************************************/

package com.vmware.bdd.service.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.vmware.aurora.vc.VcCluster;
import com.vmware.bdd.apitypes.*;
import com.vmware.bdd.exception.ClusterConfigException;
import com.vmware.bdd.manager.*;
import com.vmware.bdd.service.resmgmt.IVcInventorySyncService;
import com.vmware.bdd.service.resmgmt.sync.filter.VcResourceFilters;
import com.vmware.bdd.spectypes.*;
import com.vmware.bdd.spectypes.DiskSpec;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;

import com.google.gson.Gson;
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.vmware.aurora.composition.CreateVMFolderSP;
import com.vmware.aurora.composition.DeleteVMFolderSP;
import com.vmware.aurora.composition.DiskSchema;
import com.vmware.aurora.composition.DiskSchema.Disk;
import com.vmware.aurora.composition.NetworkSchema.Network;
import com.vmware.aurora.composition.VmSchema;
import com.vmware.aurora.composition.concurrent.ExecutionResult;
import com.vmware.aurora.composition.concurrent.Scheduler;
import com.vmware.aurora.global.Configuration;
import com.vmware.aurora.util.worker.CmsWorker;
import com.vmware.aurora.vc.*;
import com.vmware.aurora.vc.vcevent.VcEventRouter;
import com.vmware.aurora.vc.vcservice.VcContext;
import com.vmware.aurora.vc.vcservice.VcSession;
import com.vmware.bdd.apitypes.ClusterCreate;
import com.vmware.bdd.apitypes.IpBlock;
import com.vmware.bdd.apitypes.NetworkAdd;
import com.vmware.bdd.apitypes.NodeGroupCreate;
import com.vmware.bdd.apitypes.Priority;
import com.vmware.bdd.apitypes.StorageRead.DiskType;
import com.vmware.bdd.clone.spec.VmCreateResult;
import com.vmware.bdd.clone.spec.VmCreateSpec;
import com.vmware.bdd.dal.IResourcePoolDAO;
import com.vmware.bdd.entity.ClusterEntity;
import com.vmware.bdd.entity.NicEntity;
import com.vmware.bdd.entity.NodeEntity;
import com.vmware.bdd.entity.resmgmt.ResourceReservation;
import com.vmware.bdd.exception.BddException;
import com.vmware.bdd.exception.ClusteringServiceException;
import com.vmware.bdd.exception.VcProviderException;
import com.vmware.bdd.manager.intf.IClusterEntityManager;
import com.vmware.bdd.manager.intf.IConcurrentLockedClusterEntityManager;
import com.vmware.bdd.placement.Container;
import com.vmware.bdd.placement.entity.AbstractDatacenter.AbstractHost;
import com.vmware.bdd.placement.entity.BaseNode;
import com.vmware.bdd.placement.exception.PlacementException;
import com.vmware.bdd.placement.interfaces.IPlacementService;
import com.vmware.bdd.placement.util.ContainerToStringHelper;
import com.vmware.bdd.placement.util.PlacementUtil;
import com.vmware.bdd.service.ClusterNodeUpdator;
import com.vmware.bdd.service.IClusterInitializerService;
import com.vmware.bdd.service.IClusteringService;
import com.vmware.bdd.service.event.VmEventManager;
import com.vmware.bdd.service.job.NodeOperationStatus;
import com.vmware.bdd.service.job.StatusUpdater;
import com.vmware.bdd.service.resmgmt.INetworkService;
import com.vmware.bdd.service.resmgmt.INodeTemplateService;
import com.vmware.bdd.service.resmgmt.IResourceService;
import com.vmware.bdd.service.sp.*;
import com.vmware.bdd.service.utils.VcResourceUtils;
import com.vmware.bdd.software.mgmt.plugin.intf.SoftwareManager;
import com.vmware.bdd.specpolicy.GuestMachineIdSpec;
import com.vmware.bdd.utils.AuAssert;
import com.vmware.bdd.utils.ClusterUtil;
import com.vmware.bdd.utils.CommonUtil;
import com.vmware.bdd.utils.ConfigInfo;
import com.vmware.bdd.utils.Constants;
import com.vmware.bdd.utils.JobUtils;
import com.vmware.bdd.utils.VcVmUtil;
import com.vmware.bdd.utils.Version;
import com.vmware.bdd.vmclone.service.intf.IClusterCloneService;
import com.vmware.vim.binding.vim.Folder;
import com.vmware.vim.binding.vim.vm.device.VirtualDevice;
import com.vmware.vim.binding.vim.vm.device.VirtualDisk;
import com.vmware.vim.binding.vim.vm.device.VirtualDiskOption.DiskMode;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ClusteringService implements IClusteringService {
   private static final int VC_RP_MAX_NAME_LENGTH = 80;
   private static final Logger logger = Logger
         .getLogger(ClusteringService.class);
   private ClusterConfigManager configMgr;

   private IClusterEntityManager clusterEntityMgr;
   private IConcurrentLockedClusterEntityManager lockClusterEntityMgr;
   private IResourcePoolDAO rpDao;
   private INetworkService networkMgr;
   private IResourceService resMgr;
   private IPlacementService placementService;
   private IClusterInitializerService clusterInitializerService;
   private ElasticityScheduleManager elasticityScheduleMgr;
   @Autowired
   private INodeTemplateService nodeTemplateService;

   private volatile boolean inited = false;
   private volatile Throwable initError;

   private int cloneConcurrency;
   private VmEventManager processor;

   private Map<String, IClusterCloneService> cloneServiceMap;

   private ClusterManager clusterManager;

   private SoftwareManagerCollector softwareManagerCollector;

   @Autowired
   private ClusterNodeUpdator clusterNodeUpdator;

   @Autowired
   private CmsWorker cmsWorker;

   @Autowired
   private IVcInventorySyncService syncService;

   @Autowired
   private VcResourceManager vcResourceManager;

   @Autowired
   private VcResourceFilterBuilder vcResourceFilterBuilder;

   @Autowired
   public void setClusterManager(ClusterManager clusterManager) {
      this.clusterManager = clusterManager;
   }

   public ClusterManager getClusterManager() {
      return this.clusterManager;
   }

   public INetworkService getNetworkMgr() {
      return networkMgr;
   }

   public void setNetworkMgr(INetworkService networkMgr) {
      this.networkMgr = networkMgr;
   }

   public ClusterConfigManager getConfigMgr() {
      return configMgr;
   }

   public void setConfigMgr(ClusterConfigManager configMgr) {
      this.configMgr = configMgr;
   }

   @Autowired
   public void setResMgr(IResourceService resMgr) {
      this.resMgr = resMgr;
   }

   @Autowired
   public void setPlacementService(IPlacementService placementService) {
      this.placementService = placementService;
   }

   public IClusterInitializerService getClusterInitializerService() {
      return clusterInitializerService;
   }

   @Autowired
   public void setClusterInitializerService(
         IClusterInitializerService clusterInitializerService) {
      this.clusterInitializerService = clusterInitializerService;
   }

   public IClusterEntityManager getClusterEntityMgr() {
      return clusterEntityMgr;
   }

   @Autowired
   public void setClusterEntityMgr(IClusterEntityManager clusterEntityMgr) {
      this.clusterEntityMgr = clusterEntityMgr;
   }

   public IConcurrentLockedClusterEntityManager getLockClusterEntityMgr() {
      return lockClusterEntityMgr;
   }

   @Autowired
   public void setLockClusterEntityMgr(
         IConcurrentLockedClusterEntityManager lockClusterEntityMgr) {
      this.lockClusterEntityMgr = lockClusterEntityMgr;
   }

   public IResourcePoolDAO getRpDao() {
      return rpDao;
   }

   @Autowired
   public void setRpDao(IResourcePoolDAO rpDao) {
      this.rpDao = rpDao;
   }

   public Map<String, IClusterCloneService> getCloneService() {
      return cloneServiceMap;
   }

   /**
    *
    * @param cloneServiceMap clone service qualifier to clone service map.
    * 3 clone services so far: simpleClusterCloneService, fastClusterCloneService, instantClusterCloneService
    */
   @Autowired
   public void setCloneService(Map<String, IClusterCloneService> cloneServiceMap) {
      this.cloneServiceMap = cloneServiceMap;
   }

   public ElasticityScheduleManager getElasticityScheduleManager() {
      return elasticityScheduleMgr;
   }

   @Autowired
   public void setElasticityScheduleManager(
         ElasticityScheduleManager elasticityScheduleMgr) {
      this.elasticityScheduleMgr = elasticityScheduleMgr;
   }

   public SoftwareManagerCollector getSoftwareManagerCollector() {
      return softwareManagerCollector;
   }

   @Autowired
   public void setSoftwareManagerCollector(
         SoftwareManagerCollector softwareManagerCollector) {
      this.softwareManagerCollector = softwareManagerCollector;
   }

   @Override
   public boolean isInited() {
      return inited;
   }

   @Override
   public Throwable getInitError() {
      return initError;
   }

   @Override
   public synchronized void init() {
      if (!inited) {
         try {
            // XXX hack to approve bootstrap instance id, should be moved out of Configuration
            Configuration.approveBootstrapInstanceId(Configuration.BootstrapUsage.ALLOWED);
            Configuration.approveBootstrapInstanceId(Configuration.BootstrapUsage.FINALIZED);

            VcContext.initVcContext();
            new VcEventRouter();

            //disable background VC refresh.
            //CmsWorker.addPeriodic(new VcInventory.SyncInventoryRequest());
            VcInventory.loadInventory();

            //why??
            try {
               Thread.sleep(1000);
            } catch (InterruptedException e) {
               logger.warn("interupted during sleep " + e.getMessage());
            }
            startVMEventProcessor();

            int poolSize = Configuration.getInt("serengeti.scheduler.poolsize", Constants.DEFAULT_SCHEDULER_POOL_SIZE);
            Scheduler.init(poolSize, poolSize);

            cloneConcurrency = Configuration.getInt("serengeti.singlevm.concurrency", 1);

            // refresh the cluster nodes once on bde startup,
            // then add the periodic processing with default 5 minute interval
            clusterNodeUpdator.syncAllClusters();

            // initialize uuid
            initUUID();

            // refresh node templates on BDE startup
            nodeTemplateService.refreshNodeTemplates();

            clusterInitializerService.transformClusterStatus();
            elasticityScheduleMgr.start();
            configureAlarm();
            inited = true;
         } catch (Throwable err) {
            logger.error("init ClusteringService error", err);
            initError = err;
         }
      }
   }

   private void startVMEventProcessor() {
      // add event handler for Serengeti after VC event handler is registered.
      processor = new VmEventManager(lockClusterEntityMgr);
      processor.setClusterManager(clusterManager);
      processor.setSoftwareManagerCollector(softwareManagerCollector);
      processor.start();
   }

   private void configureAlarm() {
      VcDatacenter dc = VcResourceUtils.getCurrentDatacenter();
      List<String> folderList = new ArrayList<String>(1);
      String serengetiUUID = ConfigInfo.getSerengetiRootFolder();
      folderList.add(serengetiUUID);
      Folder rootFolder =
            VcResourceUtils.findFolderByNameList(dc, folderList);

      if (rootFolder == null) {
         CreateVMFolderSP sp =
               new CreateVMFolderSP(dc, null, folderList);
         Callable<Void>[] storeProcedures = new Callable[1];
         storeProcedures[0] = sp;
         Map<String, Folder> folders =
               executeFolderCreationProcedures(null, storeProcedures);
         AuAssert.check(folders.size() == 1);
         rootFolder = folders.get(serengetiUUID);
         AuAssert.check(rootFolder != null);
      }

      final Folder root = rootFolder;
      VcContext.inVcSessionDo(new VcSession<Boolean>() {
         @Override
         protected boolean isTaskSession() {
            return true;
         }

         @Override
         protected Boolean body() throws Exception {
            VcUtil.configureAlarm(root);
            return true;
         }
      });

   }

   @Override
   synchronized public void destroy() {
      Scheduler.shutdown(true);
      processor.stop();
      elasticityScheduleMgr.shutdown();
   }

   private void initUUID() {
      if (ConfigInfo.isInitUUID()) {
         String uuid = null;
         final VcVirtualMachine serverVm = VcResourceUtils.findServerVM();
         if (ConfigInfo.isDeployAsVApp()) {
            VcResourcePool vApp = serverVm.getParentVApp();
            uuid = vApp.getName();
         } else {
            Folder parentFolder = VcResourceUtils.findParentFolderOfVm(serverVm);
            AuAssert.check(parentFolder != null);
            uuid = parentFolder.getName();
         }

         ConfigInfo.setSerengetiUUID(uuid);
         ConfigInfo.setInitUUID(false);
         ConfigInfo.save();
      }
   }

   private VcVirtualMachine getTemplateVM(String templateName) {
      return this.nodeTemplateService.getNodeTemplateVMByName(templateName);
   }

   private VcVirtualMachine prepareTemplateVM(String templateName) {
      VcVirtualMachine templateVM = getTemplateVM(templateName);
      // update vm info in vc cache, in case snapshot is removed by others
      VcVmUtil.updateVm(templateVM.getId());

      try {
         boolean needRemoveSnapshots = false;
         if (ConfigInfo.isJustUpgraded()) {
            needRemoveSnapshots = true;
            ConfigInfo.setJustUpgraded(false);
            ConfigInfo.save();
         }

         if (VcVmUtil.enableOvfEnvTransport(templateVM)) {
            needRemoveSnapshots = true;
         }

         if (VcVmUtil.enableSyncTimeWithHost(templateVM)) {
            needRemoveSnapshots = true;
         }

         if (needRemoveSnapshots) {
            removeRootSnapshot(templateVM);
         }
         return templateVM;
      } catch (Exception e) {
         logger.error("Prepare template VM error: " + e.getMessage());
         throw BddException.INTERNAL(e, "Prepare template VM error.");
      }
   }

   private void removeRootSnapshot(final VcVirtualMachine templateVM) {
      VcContext.inVcSessionDo(new VcSession<Boolean>() {
         @Override
         protected boolean isTaskSession() {
            return true;
         }

         @Override
         protected Boolean body() throws Exception {
            VcSnapshot snapshot =
                  templateVM.getSnapshotByName(Constants.ROOT_SNAPSTHOT_NAME);
            if (snapshot != null) {
               snapshot.remove();
            }
            return true;
         }
      });
   }

   private BaseNode createBaseNodeFromTemplateVm(final VcVirtualMachine templateVm) {
      BaseNode templateNode = new BaseNode(templateVm.getName());
      List<DiskSpec> diskSpecs = new ArrayList<DiskSpec>();
      for (DeviceId slot : templateVm.getVirtualDiskIds()) {
         VirtualDisk vmdk = (VirtualDisk) templateVm.getVirtualDevice(slot);
         DiskSpec spec = new DiskSpec();
         spec.setSize((int) (vmdk.getCapacityInKB() / (1024 * 1024)));
         spec.setDiskType(DiskType.SYSTEM_DISK);
         spec.setController(CommonUtil.getSystemAndSwapControllerType());
         diskSpecs.add(spec);
      }
      templateNode.setDisks(diskSpecs);
      return templateNode;
   }

   private String getNodeTemplateNetworkLable(final VcVirtualMachine templateVm) {
      return VcContext.inVcSessionDo(new VcSession<String>() {
         @Override
         protected String body() throws Exception {
            VirtualDevice[] devices =
                  templateVm.getConfig().getHardware().getDevice();
            for (VirtualDevice device : devices) {
               if (device.getKey() == 4000) {
                  return device.getDeviceInfo().getLabel();
               }
            }
            return null;
         }
      });
   }

   private void updateNicLabels(List<BaseNode> vNodes, String templateNetworkLabel) {
      logger.info("Start to set network schema for template nic.");
      for (BaseNode node : vNodes) {
         List<Network> networks = node.getVmSchema().networkSchema.networks;
         if (!networks.isEmpty()) {
            networks.get(0).nicLabel = templateNetworkLabel;
            for (int i = 1; i < networks.size(); i++) {
               networks.get(i).nicLabel = VcVmUtil.NIC_LABEL_PREFIX + (i + 1);
            }
         }
         logger.info(node.getVmSchema());
      }
   }

   /**
    * cluster create, resize, resume will all call this method for static ip
    * allocation the network contains all allocated ip address to this cluster,
    * so some of them may already be occupied by existing node. So we need to
    * detect if that ip is allocated, before assign that one to one node
    *
    * @param vNodes
    * @param networkAdds
    * @param occupiedIpSets
    */
   private void allocateStaticIp(List<BaseNode> vNodes,
         List<NetworkAdd> networkAdds, Map<String, Set<String>> occupiedIpSets) {
      int i, j;
      for (i = 0; i < networkAdds.size(); i++) {
         NetworkAdd networkAdd = networkAdds.get(i);
         String portGroupName = networkAdd.getPortGroup();
         Set<String> usedIps = null;
         if (occupiedIpSets != null && !occupiedIpSets.isEmpty()) {
            usedIps = occupiedIpSets.get(portGroupName);
         }

         if (networkAdd.getIsDhcp()) {
            // no need to allocate ip for dhcp
            logger.info("using dhcp for network: " + portGroupName);
         } else {
            logger.info("Start to allocate static ip address for each VM's "
                  + i + "th network.");
            List<String> availableIps =
                  IpBlock.getIpAddressFromIpBlock(networkAdd.getIpBlocks());
            if (usedIps != null && !usedIps.isEmpty()) {
               availableIps.removeAll(usedIps);
            }
            AuAssert.check(availableIps.size() == vNodes.size());
            for (j = 0; j < availableIps.size(); j++) {
               vNodes.get(j).updateNicOfPortGroup(portGroupName,
                     availableIps.get(j), null, null);
            }
            logger.info("Finished to allocate static ip address for VM's mgr network.");
         }
      }
   }

   private String getMaprActiveJobTrackerIp(final String maprNodeIP,
         final String clusterName) {
      String activeJobTrackerIp = "";
      String errorMsg = "";
      JSch jsch = new JSch();
      String sshUser = Configuration.getString("mapr.ssh.user", "serengeti");
      int sshPort = Configuration.getInt("mapr.ssh.port", 22);
      String sudoCmd = CommonUtil.getCustomizedSudoCmd();

      String prvKeyFile =
            Configuration.getString("serengeti.ssh.private.key.file",
                  "/home/serengeti/.ssh/id_rsa");
      Session session = null;
      ChannelExec channel = null;
      BufferedReader in = null;
      try {
         session = jsch.getSession(sshUser, maprNodeIP, sshPort);
         jsch.addIdentity(prvKeyFile);
         java.util.Properties config = new java.util.Properties();
         config.put("StrictHostKeyChecking", "no");
         session.setConfig(config);
         session.setTimeout(15000);
         session.connect();
         logger.debug("SSH session is connected!");
         channel = (ChannelExec) session.openChannel("exec");
         if (channel != null) {
            logger.debug("SSH channel is connected!");
            StringBuffer buff = new StringBuffer();
            String cmd =
                  "maprcli node list -filter \"[rp==/*]and[svc==jobtracker]\" -columns ip";
            logger.debug("exec command is: " + cmd);
            channel.setPty(true); //to enable sudo
            channel.setCommand(sudoCmd + " " + cmd);
            in =
                  new BufferedReader(new InputStreamReader(
                        channel.getInputStream()));
            channel.connect();
            if (!canChannelConnect(channel)) {
               errorMsg =
                     "Cannot determine IP address of active JobTracker. No SSH channel connection.";
               logger.error(errorMsg);
               throw BddException.INTERNAL(null, errorMsg);
            }
            while (true) {
               String line = in.readLine();
               buff.append(line);
               logger.debug("jobtracker message: " + line);
               if (channel.isClosed()) {
                  int exitStatus = channel.getExitStatus();
                  logger.debug("Exit status from exec is: " + exitStatus);
                  break;
               }
            }
            Pattern ipPattern = Pattern.compile(Constants.IP_PATTERN);
            Matcher matcher = ipPattern.matcher(buff.toString());
            if (matcher.find()) {
               activeJobTrackerIp = matcher.group();
            } else {
               errorMsg =
                     "Cannot find jobtracker ip info in cluster" + clusterName;
               logger.error(errorMsg);
               throw BddException.INTERNAL(null, errorMsg);
            }
         } else {
            errorMsg = "Get active Jobtracker ip: cannot open SSH channel.";
            logger.error(errorMsg);
            throw BddException.INTERNAL(null, errorMsg);
         }
      } catch (JSchException e) {
         errorMsg = "SSH unknow error: " + e.getMessage();
         logger.error(errorMsg);
         throw BddException.INTERNAL(null, errorMsg);
      } catch (IOException e) {
         errorMsg = "Obtain active jobtracker ip error: " + e.getMessage();
         logger.error(errorMsg);
         throw BddException.INTERNAL(null, errorMsg);
      } finally {
         if (channel != null && channel.isConnected()) {
            channel.disconnect();
         }
         if (session != null && session.isConnected()) {
            session.disconnect();
         }
         if (in != null) {
            try {
               in.close();
            } catch (IOException e) {
               errorMsg =
                     "Failed to release buffer while retrieving MapR JobTracker Port: "
                           + e.getMessage();
               logger.error(errorMsg);
            }
         }
      }
      return activeJobTrackerIp;
   }

   private boolean canChannelConnect(ChannelExec channel) {
      if (channel == null) {
         return false;
      }
      if (channel.isConnected()) {
         return true;
      }
      try {
         channel.connect();
      } catch (JSchException e) {
         String errorMsg = "SSH connection failed: " + e.getMessage();
         logger.error(errorMsg);
         throw BddException.INTERNAL(null, errorMsg);
      }
      return channel.isConnected();
   }

   private void updateVhmMasterMoid(String clusterName) {
      ClusterEntity cluster = getClusterEntityMgr().findByName(clusterName);
      if (cluster.getVhmMasterMoid() == null) {
         List<NodeEntity> nodes =
               getClusterEntityMgr().findAllNodes(clusterName);
         for (NodeEntity node : nodes) {
            if (node.getMoId() != null
                  && node.getNodeGroup().getRoles() != null) {
               @SuppressWarnings("unchecked")
               List<String> roles =
                     new Gson().fromJson(node.getNodeGroup().getRoles(),
                           List.class);
               if (cluster.getDistroVendor().equalsIgnoreCase(Constants.MAPR_VENDOR)) {
                  if (roles
                        .contains(HadoopRole.MAPR_JOBTRACKER_ROLE.toString())) {
                     String thisJtIp = node.getPrimaryMgtIpV4();
                     String activeJtIp;
                     try {
                        activeJtIp =
                              getMaprActiveJobTrackerIp(thisJtIp, clusterName);
                        logger.info("fetched active JT Ip: " + activeJtIp);
                     } catch (Exception e) {
                        continue;
                     }

                     /*
                      * TODO: in multiple NICs env, if portgroups are isolated,
                      * may not able to retrieve active IP.
                      */
                     AuAssert.check(!CommonUtil.isBlank(thisJtIp),
                           "falied to query active JobTracker Ip");
                     for (NodeEntity jt : nodes) {
                        boolean isActiveJt = false;
                        for (NicEntity nicEntity : jt.getNics()) {
                           if (nicEntity.getIpv4Address() != null
                                 && activeJtIp.equals(nicEntity
                                       .getIpv4Address())) {
                              isActiveJt = true;
                              break;
                           }
                        }

                        if (isActiveJt) {
                           cluster.setVhmMasterMoid(jt.getMoId());
                           break;
                        }
                     }
                     break;
                  }
               } else {
                  if (roles.contains(HadoopRole.HADOOP_JOBTRACKER_ROLE
                        .toString())) {
                     cluster.setVhmMasterMoid(node.getMoId());
                     break;
                  }
               }
            }
         }
      }
      getClusterEntityMgr().update(cluster);
   }

   @Override
   @SuppressWarnings("unchecked")
   public boolean setAutoElasticity(String clusterName, boolean refreshAllNodes) {
      logger.info("set auto elasticity for cluster " + clusterName);

      ClusterEntity cluster = getClusterEntityMgr().findByName(clusterName);
      List<NodeEntity> nodes = clusterEntityMgr.findAllNodes(clusterName);

      Boolean enableAutoElasticity = cluster.getAutomationEnable();
      if (enableAutoElasticity == null) {
         return true;
      }

      String masterMoId = cluster.getVhmMasterMoid();
      if (masterMoId == null) {
         // this will only occurs when creating cluster
         updateVhmMasterMoid(clusterName);
         cluster = getClusterEntityMgr().findByName(clusterName);
         masterMoId = cluster.getVhmMasterMoid();
         if (masterMoId == null) {
            logger.error("masterMoId missed.");
            throw ClusteringServiceException.SET_AUTO_ELASTICITY_FAILED(cluster
                  .getName());
         }
      }

      String serengetiUUID = ConfigInfo.getSerengetiRootFolder();
      int minComputeNodeNum = cluster.getVhmMinNum();
      int maxComputeNodeNum = cluster.getVhmMaxNum();
      String jobTrackerPort = cluster.getVhmJobTrackerPort();

      VcVirtualMachine vcVm = VcCache.getIgnoreMissing(masterMoId);
      if (vcVm == null) {
         logger.error("cannot find vhm master node");
         return false;
      }
      String masterUUID = vcVm.getConfig().getUuid();

      Callable<Void>[] storeProcedures = new Callable[nodes.size()];
      int i = 0;
      for (NodeEntity node : nodes) {
         VcVirtualMachine vm = VcCache.getIgnoreMissing(node.getMoId());
         if (vm == null) {
            logger.error("cannot find node: " + node.getVmName());
            continue;
         }
         if (!refreshAllNodes && !vm.getId().equalsIgnoreCase(masterMoId)) {
            continue;
         }
         List<String> roles =
               new Gson().fromJson(node.getNodeGroup().getRoles(), List.class);
         SoftwareManager softMgr =
            softwareManagerCollector.getSoftwareManager(cluster.getAppManager());

         boolean isComputeOnlyNode =
            softMgr.isComputeOnlyRoles(roles);
         SetAutoElasticitySP sp =
               new SetAutoElasticitySP(clusterName, vm, serengetiUUID,
                     masterMoId, masterUUID, enableAutoElasticity,
                     minComputeNodeNum, maxComputeNodeNum, jobTrackerPort,
                     isComputeOnlyNode);
         storeProcedures[i] = sp;
         i++;
      }
      try {
         // execute store procedures to set auto elasticity
         logger.info("ClusteringService, start to set auto elasticity.");
         boolean success = true;
         NoProgressUpdateCallback callback = new NoProgressUpdateCallback();
         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storeProcedures, callback);
         if (result == null) {
            logger.error("set auto elasticity failed.");
            throw ClusteringServiceException
                  .SET_AUTO_ELASTICITY_FAILED(clusterName);
         }
         for (i = 0; i < storeProcedures.length; i++) {
            if (result[i].throwable != null) {
               logger.error("failed to set auto elasticity",
                     result[i].throwable);
               success = false;
            }
         }
         return success;
      } catch (InterruptedException e) {
         logger.error("error in setting auto elasticity", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   @SuppressWarnings("unchecked")
   private Map<String, Folder> createVcFolders(ClusterCreate cluster) {
      return createVcFolders(cluster, false);
   }

   /*
    * addNodeGroup means  it's called by expandCluster() to add new node groups
    */
   @SuppressWarnings("unchecked")
   private Map<String, Folder> createVcFolders(ClusterCreate cluster, boolean addNodeGroup) {
      logger.info("createVcFolders, start to create cluster Folder.");
      VcVirtualMachine templateVm = getTemplateVM(cluster.getTemplateName());
      // get all nodegroups
      Callable<Void>[] storeProcedures = new Callable[1];
      Folder clusterFolder = null;

      if(!addNodeGroup) {
         if (cluster.getNodeGroups().length > 0) {
            // create cluster folder first
            NodeGroupCreate group = cluster.getNodeGroups()[0];
            String path = group.getVmFolderPath();
            logger.info("create folderName" + path);
            String[] folderNames = path.split("/");
            List<String> folderList = new ArrayList<String>();
            for (int i = 0; i < folderNames.length - 1; i++) {
               folderList.add(folderNames[i]);
            }
            CreateVMFolderSP sp =
                    new CreateVMFolderSP(templateVm.getDatacenter(), null,
                            folderList);
            storeProcedures[0] = sp;
            Map<String, Folder> folders =
                    executeFolderCreationProcedures(cluster, storeProcedures);
            for (String name : folders.keySet()) {
               clusterFolder = folders.get(name);
               break;
            }
         }
      }

      logger.info("createVcFolders, start to create group Folders.");
      storeProcedures = new Callable[cluster.getNodeGroups().length];
      int i = 0;
      for (NodeGroupCreate group : cluster.getNodeGroups()) {
         List<String> folderList = new ArrayList<String>();
         folderList.add(group.getName());
         CreateVMFolderSP sp =
               new CreateVMFolderSP(templateVm.getDatacenter(), clusterFolder,
                     folderList);
         storeProcedures[i] = sp;
         i++;
      }
      return executeFolderCreationProcedures(cluster, storeProcedures);
   }

   private Map<String, Folder> executeFolderCreationProcedures(
         ClusterCreate cluster, Callable<Void>[] storeProcedures) {
      Map<String, Folder> folders = new HashMap<String, Folder>();
      try {
         // execute store procedures to create vc folders
         NoProgressUpdateCallback callback = new NoProgressUpdateCallback();
         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storeProcedures, callback);
         if (result == null) {
            logger.error("No folder is created.");
            if (cluster != null)
               throw ClusteringServiceException.CREATE_FOLDER_FAILED(cluster
                     .getName());
         }

         int total = 0;
         boolean success = true;
         for (int i = 0; i < storeProcedures.length; i++) {
            CreateVMFolderSP sp = (CreateVMFolderSP) storeProcedures[i];
            if (result[i].finished && result[i].throwable == null) {
               ++total;
               Folder childFolder =
                     sp.getResult().get(sp.getResult().size() - 1);
               folders.put(childFolder.getName(), childFolder);
            } else if (result[i].throwable != null) {
               logger.error("Failed to create vm folder", result[i].throwable);
               success = false;
            }
         }
         logger.info(total + " Folders are created.");
         if (!success) {
            if (cluster != null)
               throw ClusteringServiceException.CREATE_FOLDER_FAILED(cluster
                     .getName());
         }
         return folders;
      } catch (InterruptedException e) {
         logger.error("error in creating VC folders", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }


   private void executeResourcePoolStoreProcedures(Callable<Void>[] defineSPs,
         String type, String clusterName) throws InterruptedException {
      if (defineSPs.length == 0) {
         logger.debug("no resource pool need to be created.");
         return;
      }
      NoProgressUpdateCallback callback = new NoProgressUpdateCallback();
      ExecutionResult[] result =
            Scheduler.executeStoredProcedures(
                  com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                  defineSPs, callback);
      if (result == null) {
         logger.error("No " + type + " resource pool is created.");
         throw ClusteringServiceException
               .CREATE_RESOURCE_POOL_FAILED(Constants.NO_RESOURCE_POOL_IS_CREATED);
      }
      int total = 0;
      boolean success = true;
      String errMessage = null;
      for (int i = 0; i < defineSPs.length; i++) {
         if (result[i].finished && result[i].throwable == null) {
            ++total;
         } else if (result[i].throwable != null) {
            if (errMessage == null) {
               //Generally the error message is same for all resource pools, so here we just keep the first failure message.
               errMessage = result[i].throwable.getMessage();
            }
            logger.error("Failed to create " + type + " resource pool(s)",
                  result[i].throwable);
            success = false;
         }
      }
      logger.info(total + " " + type + " resource pool(s) are created.");
      if (!success) {
         throw ClusteringServiceException
               .CREATE_RESOURCE_POOL_FAILED(errMessage);
      }
   }

   private Map<String, Integer> collectResourcePoolInfo(List<BaseNode> vNodes,
         final String uuid, Map<String, List<String>> vcClusterRpNamesMap,
         Map<Long, List<NodeGroupCreate>> rpNodeGroupsMap) {
      List<String> resourcePoolNames = null;
      List<NodeGroupCreate> nodeGroups = null;
      int resourcePoolNameCount = 0;
      int nodeGroupNameCount = 0;
      for (BaseNode baseNode : vNodes) {
         String vcCluster = baseNode.getTargetVcCluster();
         VcCluster cluster = VcResourceUtils.findVcCluster(vcCluster);
         if (!cluster.getConfig().getDRSEnabled()) {
            logger.debug("DRS disabled for cluster " + vcCluster
                  + ", do not create child rp for this cluster.");
            continue;
         }
         AuAssert.check(!CommonUtil.isBlank(vcCluster),
               "Vc cluster name cannot be null!");
         if (!vcClusterRpNamesMap.containsKey(vcCluster)) {
            resourcePoolNames = new ArrayList<String>();
         } else {
            resourcePoolNames = vcClusterRpNamesMap.get(vcCluster);
         }
         String vcRp = baseNode.getTargetRp();
         logger.info("collectResourcePoolInfo baseNode.getTargetRp()" + vcRp);
         String rpPath = "/" + vcCluster + "/" + vcRp + "/" + uuid;
         logger.info("collectResourcePoolInfo rpPath" + rpPath);
         long rpHashCode = rpPath.hashCode();
         if (!rpNodeGroupsMap.containsKey(rpHashCode)) {
            nodeGroups = new ArrayList<NodeGroupCreate>();
         } else {
            nodeGroups = rpNodeGroupsMap.get(rpHashCode);
         }
         NodeGroupCreate nodeGroup = baseNode.getNodeGroup();
         if (!getAllNodeGroupNames(nodeGroups).contains(nodeGroup.getName())) {
            nodeGroups.add(nodeGroup);
            rpNodeGroupsMap.put(rpHashCode, nodeGroups);
            nodeGroupNameCount++;
         }
         if (!resourcePoolNames.contains(vcRp)) {
            resourcePoolNames.add(vcRp);
            vcClusterRpNamesMap.put(vcCluster, resourcePoolNames);
            resourcePoolNameCount++;
         }
      }

      Map<String, Integer> countResult = new HashMap<String, Integer>();
      countResult.put("resourcePoolNameCount", resourcePoolNameCount);
      countResult.put("nodeGroupNameCount", nodeGroupNameCount);
      return countResult;
   }

   private List<String> getAllNodeGroupNames(List<NodeGroupCreate> nodeGroups) {
      List<String> nodeGroupNames = new ArrayList<String>();
      for (NodeGroupCreate nodeGroup : nodeGroups) {
         nodeGroupNames.add(nodeGroup.getName());
      }
      return nodeGroupNames;
   }

   private String createVcResourcePools(List<BaseNode> vNodes) {
      return createVcResourcePools(vNodes, false);
   }

   /*
    * addNodeGroup means  it's called by expandCluster() to add new node groups
    */
   private String createVcResourcePools(List<BaseNode> vNodes, boolean addNodeGroup) {
      logger.info("createVcResourcePools, start to create VC ResourcePool(s).");
      /*
       * define cluster resource pool name.
       */
      String clusterName = vNodes.get(0).getClusterName();
      SoftwareManager softManager =
            softwareManagerCollector
                  .getSoftwareManagerByClusterName(clusterName);
      String uuid = ConfigInfo.getSerengetiUUID();
      String clusterRpName = uuid + "-" + clusterName;
      if (clusterRpName.length() > VC_RP_MAX_NAME_LENGTH) {
         throw ClusteringServiceException.CLUSTER_NAME_TOO_LONG(clusterName);
      }

      /*
       * prepare resource pool names and node group per resource pool for creating cluster
       * resource pools and node group resource pool(s).
       */
      Map<String, List<String>> vcClusterRpNamesMap =
            new HashMap<String, List<String>>();
      Map<Long, List<NodeGroupCreate>> rpNodeGroupsMap =
            new HashMap<Long, List<NodeGroupCreate>>();
      Map<String, Integer> countResult =
            collectResourcePoolInfo(vNodes, uuid, vcClusterRpNamesMap,
                  rpNodeGroupsMap);

      try {
         int i = 0;
         if (!addNodeGroup) {
                     /*
          * define cluster store procedures of resource pool(s)
          */
            int resourcePoolNameCount = countResult.get("resourcePoolNameCount");
            Callable<Void>[] clusterSPs = new Callable[resourcePoolNameCount];
            for (Entry<String, List<String>> vcClusterRpNamesEntry : vcClusterRpNamesMap
                    .entrySet()) {
               String vcClusterName = vcClusterRpNamesEntry.getKey();
               VcCluster vcCluster = VcResourceUtils.findVcCluster(vcClusterName);
               if (vcCluster == null) {
                  String errorMsg =
                          "Cannot find the vCenter Server cluster " + vcClusterName
                                  + ".";
                  logger.error(errorMsg);
                  throw ClusteringServiceException
                          .CANNOT_FIND_VC_CLUSTER(vcClusterName);
               }
               List<String> resourcePoolNames = vcClusterRpNamesEntry.getValue();
               for (String resourcePoolName : resourcePoolNames) {
                  VcResourcePool parentVcResourcePool =
                          VcResourceUtils.findRPInVCCluster(vcClusterName,
                                  resourcePoolName);
                  if (parentVcResourcePool == null) {
                     String errorMsg =
                             "Cannot find the vCenter Server resource pool "
                                     + resourcePoolName + ".";
                     logger.error(errorMsg);
                     throw ClusteringServiceException
                             .CANNOT_FIND_VC_RESOURCE_POOL(resourcePoolName);
                  }
                  CreateResourcePoolSP clusterSP =
                          new CreateResourcePoolSP(parentVcResourcePool,
                                  clusterRpName);
                  clusterSPs[i] = clusterSP;
                  i++;
               }
            }

            // execute store procedures to create cluster resource pool(s)
            logger.info("ClusteringService, start to create cluster resource pool(s).");
            executeResourcePoolStoreProcedures(clusterSPs, "cluster", clusterName);

         }

         /*
          * define node group store procedures of resource pool(s)
          */
         int nodeGroupNameCount = countResult.get("nodeGroupNameCount");
         Callable<Void>[] nodeGroupSPs = new Callable[nodeGroupNameCount];
         i = 0;
         for (Entry<String, List<String>> vcClusterRpNamesEntry : vcClusterRpNamesMap
               .entrySet()) {
            String vcClusterName = vcClusterRpNamesEntry.getKey();
            VcCluster vcCluster = VcResourceUtils.findVcCluster(vcClusterName);
            if (vcCluster == null) {
               String errorMsg =
                     "Cannot find the vCenter Server cluster " + vcClusterName
                           + ".";
               logger.error(errorMsg);
               throw ClusteringServiceException
                     .CANNOT_FIND_VC_CLUSTER(vcClusterName);
            }
            if (!vcCluster.getConfig().getDRSEnabled()) {
               continue;
            }
            List<String> resourcePoolNames = vcClusterRpNamesEntry.getValue();
            for (String resourcePoolName : resourcePoolNames) {
               VcResourcePool parentVcResourcePool = null;
               String vcRPName =
                     CommonUtil.isBlank(resourcePoolName) ? clusterRpName
                           : resourcePoolName + "/" + clusterRpName;
               parentVcResourcePool =
                     VcResourceUtils.findRPInVCCluster(vcClusterName, vcRPName);
               if (parentVcResourcePool == null) {
                  String errorMsg =
                        "Cannot find the vCenter Server resource pool "
                              + vcRPName
                              + (CommonUtil.isBlank(resourcePoolName) ? ""
                                    : " in the vCenter Server resource pool "
                                          + resourcePoolName) + ".";
                  logger.error(errorMsg);
                  throw ClusteringServiceException
                        .CANNOT_FIND_SUB_VC_RESOURCE_POOL(vcRPName,
                              resourcePoolName);
               }
               String rpPath =
                     "/" + vcClusterName + "/" + resourcePoolName + "/" + uuid;
               long rpHashCode = rpPath.hashCode();
               for (NodeGroupCreate nodeGroup : rpNodeGroupsMap.get(rpHashCode)) {
                  AuAssert
                        .check(nodeGroup != null,
                              "create node group resource pool failed: node group cannot be null !");
                  if (nodeGroup.getName().length() > 80) {
                     throw ClusteringServiceException
                           .GROUP_NAME_TOO_LONG(nodeGroup.getName());
                  }
                  CreateResourcePoolSP nodeGroupSP =
                        new CreateResourcePoolSP(parentVcResourcePool,
                              nodeGroup.getName(), nodeGroup, softManager);
                  nodeGroupSPs[i] = nodeGroupSP;
                  i++;
               }
            }
         }

         //execute store procedures to create node group resource pool(s)
         logger.info("ClusteringService, start to create node group resource pool(s).");
         executeResourcePoolStoreProcedures(nodeGroupSPs, "node group",
               clusterName);
      } catch (InterruptedException e) {
         logger.error("error in creating VC ResourcePool(s)", e);
         throw ClusteringServiceException.CREATE_RESOURCE_POOL_ERROR(e
               .getMessage());
      }
      return clusterRpName;
   }

   @SuppressWarnings("unchecked")
   @Override
   public boolean createVcVms(ClusterCreate clusterSpec,
         List<BaseNode> vNodes, Map<String, Set<String>> occupiedIpSets,
         boolean reserveRawDisks, StatusUpdater statusUpdator) {
      if (vNodes.isEmpty()) {
         logger.info("No vm to be created.");
         return true;
      }
      List<NetworkAdd> networkAdds = clusterSpec.getNetworkings();
      String clusterCloneType = clusterSpec.getClusterCloneType();
      logger.info("syncCreateVMs, start to create VMs for cluster " + clusterSpec.getName());

      logger.info(String.format("Preparing node template VM %s for creating cluster %s", clusterSpec.getTemplateName(), clusterSpec.getName()));
      VcVirtualMachine templateVm = prepareNodeTemplate(clusterSpec.getTemplateName(), vNodes);
      logger.info("node template VM moid is " + templateVm.getId());
      VmCreateSpec sourceSpec = new VmCreateSpec();
      sourceSpec.setVmId(templateVm.getId());
      sourceSpec.setVmName(templateVm.getName());
      sourceSpec.setTargetHost(templateVm.getHost());

      allocateStaticIp(vNodes, networkAdds, occupiedIpSets);
      Map<String, Folder> folders = createVcFolders(vNodes.get(0).getCluster());
      String clusterRpName = createVcResourcePools(vNodes);

      List<VmCreateSpec> specs = new ArrayList<VmCreateSpec>();
      Map<String, BaseNode> nodeMap = new HashMap<String, BaseNode>();
      for (BaseNode vNode : vNodes) {
         // prepare for cloning result
         nodeMap.put(vNode.getVmName(), vNode);
         vNode.setSuccess(false);
         vNode.setFinished(false);
         // generate create spec for fast clone
         VmCreateSpec spec = new VmCreateSpec();
         VmSchema createSchema = getVmSchema(vNode, templateVm.getId());
         spec.setSchema(createSchema);
         GuestMachineIdSpec machineIdSpec =
               new GuestMachineIdSpec(networkAdds,
                     vNode.fetchPortGroupToIpV4Map(),
                     vNode.getPrimaryMgtPgName(), vNode, networkMgr);
         logger.info("machine id of vm " + vNode.getVmName() + ":\n"
               + machineIdSpec.toString());
         spec.setBootupConfigs(machineIdSpec.toGuestVariable());
         // timeout is 10 mintues
         StartVmPostPowerOn query =
               new StartVmPostPowerOn(vNode.getNics().keySet(),
                     Configuration.getInt(Constants.VM_POWER_ON_WAITING_SEC_KEY, Constants.VM_POWER_ON_WAITING_SEC));
         spec.setPostPowerOn(query);
         spec.setPrePowerOn(getPrePowerOnFunc(vNode, reserveRawDisks));
         spec.setCloneType(VcVmCloneType.FULL);
         spec.setTargetDs(getVcDatastore(vNode));
         spec.setTargetFolder(folders.get(vNode.getGroupName()));
         spec.setTargetHost(VcResourceUtils.findHost(vNode.getTargetHost()));
         spec.setTargetRp(getVcResourcePool(vNode, clusterRpName));
         spec.setVmName(vNode.getVmName());
         specs.add(spec);
      }

      BaseProgressCallback callback = new BaseProgressCallback(statusUpdator);

      logger.info("ClusteringService, start to clone template.");
      AuAssert.check(specs.size() > 0);
      VmSchema vmSchema = specs.get(0).getSchema();
      VcVmUtil.checkAndCreateSnapshot(vmSchema);

      // call clone service to copy templates
      List<VmCreateResult<?>> results =
            chooseClusterCloneService(clusterCloneType).createCopies(sourceSpec, cloneConcurrency, specs,
                  callback);
      if (results == null || results.isEmpty()) {
         for (VmCreateSpec spec : specs) {
            BaseNode node = nodeMap.get(spec.getVmName());
            node.setFinished(true);
            node.setSuccess(false);
         }
         return false;
      }
      boolean success = true;
      int total = 0;
      for (VmCreateResult<?> result : results) {
         VmCreateSpec spec = (VmCreateSpec) result.getSpec();
         BaseNode node = nodeMap.get(spec.getVmName());
         node.setVmMobId(spec.getVmId());
         node.setSuccess(true);
         node.setFinished(true);
         boolean vmSucc = VcVmUtil.setBaseNodeForVm(node, spec.getVmId());
         if (!vmSucc || !result.isSuccess()) {
            success = false;
            node.setSuccess(false);
            if (result.getErrMessage() != null) {
               node.setErrMessage(result.getErrTimestamp() + " "
                     + result.getErrMessage());
            } else if (!node.getErrMessage().isEmpty()) {
               node.setErrMessage(CommonUtil.getCurrentTimestamp() + " "
                     + node.getNodeAction());
            }
         } else {
            total++;
         }
      }
      logger.info(total + " VMs are successfully created.");
      return success;
   }

   private VcVirtualMachine prepareNodeTemplate(String templateName, List<BaseNode> vNodes) {
      VcVirtualMachine templateVM = prepareTemplateVM(templateName);
      String lable = getNodeTemplateNetworkLable(templateVM);
      updateNicLabels(vNodes, lable);
      return templateVM;
   }

   private IClusterCloneService chooseClusterCloneService(String type) {
      String qualifier = type+"ClusterCloneService";
      IClusterCloneService clusterCloneService = this.cloneServiceMap.get(qualifier);

      AuAssert.check(clusterCloneService != null, "ClusterCloneService not found: " + type);

      return clusterCloneService;
   }

   private void setPersistentDiskMode(BaseNode vNode) {
      DiskSchema diskSchema = vNode.getVmSchema().diskSchema;
      if (diskSchema.getDisks() != null) {
         for (Disk disk : diskSchema.getDisks()) {
            disk.mode = DiskMode.persistent;
         }
      }
   }

   private CreateVmPrePowerOn getPrePowerOnFunc(BaseNode vNode, boolean reserveRawDisks) {
      boolean persistentDiskMode = false;

      String haFlag = vNode.getNodeGroup().getHaFlag();
      boolean ha = false;
      boolean ft = false;
      if (haFlag != null && Constants.HA_FLAG_ON.equals(haFlag.toLowerCase())) {
         ha = true;
      }
      if (haFlag != null && Constants.HA_FLAG_FT.equals(haFlag.toLowerCase())) {
         ha = true;
         ft = true;
         Integer cpuNum = vNode.getNodeGroup().getCpuNum();
         cpuNum = (cpuNum == null) ? 0 : cpuNum;
         if (cpuNum > 1) {
            throw ClusteringServiceException.CPU_NUMBER_MORE_THAN_ONE(vNode
                  .getVmName());
         }

         logger.debug("ft is enabled is for VM " + vNode.getVmName());
         logger.debug("set disk mode to persistent for VM " + vNode.getVmName());
         // change disk mode to persistent, instead of independent_persistent, since FT requires this
         persistentDiskMode = true;
      }

      List<String> roles = vNode.getNodeGroup().getRoles();
      SoftwareManager softMgr =
         softwareManagerCollector.getSoftwareManagerByClusterName(vNode.getClusterName());

      if (roles != null && softMgr.hasMgmtRole(roles)) {
         logger.debug(vNode.getVmName() + " is a master node");
         logger.debug("set disk mode to persistent for VM " + vNode.getVmName());
         // change disk mode to persistent, instead of independent_persistent, to allow snapshot and clone on VM
         persistentDiskMode = true;
      }

      if (persistentDiskMode) {
         setPersistentDiskMode(vNode);
      }

      ClusterEntity clusterEntity =
            getClusterEntityMgr().findByName(vNode.getClusterName());
      CreateVmPrePowerOn prePowerOn =
            new CreateVmPrePowerOn(reserveRawDisks, vNode.getVmSchema().diskSchema.getDisks(),
                  ha, ft, clusterEntity.getIoShares());

      return prePowerOn;
   }

   private static VcDatastore getVcDatastore(BaseNode vNode) {
      VcDatastore ds = VcResourceUtils.findDSInVcByName(vNode.getTargetDs());
      if (ds != null) {
         return ds;
      }
      logger.error("target data store " + vNode.getTargetDs()
            + " is not found.");
      throw ClusteringServiceException.TARGET_VC_DATASTORE_NOT_FOUND(vNode
            .getTargetDs());
   }

   private VcResourcePool getVcResourcePool(BaseNode vNode,
         final String clusterRpName) {
      try {
         String vcRPName = "";
         VcCluster cluster =
               VcResourceUtils.findVcCluster(vNode.getTargetVcCluster());
         if (!cluster.getConfig().getDRSEnabled()) {
            logger.debug("DRS disabled for cluster "
                  + vNode.getTargetVcCluster()
                  + ", put VM under cluster directly.");
            return cluster.getRootRP();
         }
         if (CommonUtil.isBlank(vNode.getTargetRp())) {
            vcRPName = clusterRpName + "/" + vNode.getNodeGroup().getName();
         } else {
            vcRPName =
                  vNode.getTargetRp() + "/" + clusterRpName + "/"
                        + vNode.getNodeGroup().getName();
         }
         VcResourcePool rp =
               VcResourceUtils.findRPInVCCluster(vNode.getTargetVcCluster(),
                     vcRPName);
         if (rp == null) {
            throw ClusteringServiceException.TARGET_VC_RP_NOT_FOUND(
                  vNode.getTargetVcCluster(), vNode.getTargetRp());
         }
         return rp;
      } catch (Exception e) {
         logger.error("Failed to get VC resource pool " + vNode.getTargetRp()
               + " in vc cluster " + vNode.getTargetVcCluster(), e);

         throw ClusteringServiceException.TARGET_VC_RP_NOT_FOUND(
               vNode.getTargetVcCluster(), vNode.getTargetRp());
      }
   }

   private VmSchema getVmSchema(BaseNode vNode, String templateVmId) {
      VmSchema schema = vNode.getVmSchema();
      schema.diskSchema.setParent(templateVmId);
      schema.diskSchema.setParentSnap(Constants.ROOT_SNAPSTHOT_NAME);
      return schema;
   }

   @Override
   public List<BaseNode> getPlacementPlan(ClusterCreate clusterSpec,
         List<BaseNode> existedNodes) {

      try {
         Thread.sleep(Configuration.getInt("vcrefresh.delay.forResume", 10) * 1000);
      } catch (InterruptedException e1) {
         logger.error("vcrefresh.delay.forResume interrupted.", e1);
      }

      List<String> dsNames = vcResourceManager.getDsNamesToBeUsed(clusterSpec.getDsNames());
      if (dsNames.isEmpty()) {
         throw ClusterConfigException.NO_DATASTORE_ADDED();
      }

      VcResourceFilters filters = vcResourceFilterBuilder.build(dsNames,
            vcResourceManager.getRpNames(clusterSpec.getRpNames()), clusterSpec.getNetworkNames());

      try {
         syncService.refreshInventory(filters);
      } catch (InterruptedException e) {
         logger.error("failed to refresh vc inventory.", e);
      }

      logger.info("Begin to calculate provision plan.");

      logger.info("Calling resource manager to get available vc hosts");
      Container container = new Container();

      List<VcCluster> clusters = resMgr.getAvailableClusters();
      AuAssert.check(clusters != null && clusters.size() != 0);
      for (VcCluster cl : clusters) {
         //refresh once at beginning of cluster create/resume/resize
//         VcResourceUtils.refreshDatastore(cl);
         container.addResource(cl);
      }

      logger.info("VC resource container details:" + ContainerToStringHelper.convertToString(container));


      logger.info("check time on hosts.");
      // check time on hosts
      int maxTimeDiffInSec = Constants.MAX_TIME_DIFF_IN_SEC;
      SoftwareManager softMgr =
            softwareManagerCollector.getSoftwareManager(clusterSpec
                  .getAppManager());
      if (softMgr.hasHbase(clusterSpec.toBlueprint()))
         maxTimeDiffInSec = Constants.MAX_TIME_DIFF_IN_SEC_HBASE;

      List<String> outOfSyncHosts = new ArrayList<String>();
      for (AbstractHost host : container.getAllHosts()) {
         int hostTimeDiffInSec =
               VcResourceUtils.getHostTimeDiffInSec(host.getName());
         if (Math.abs(hostTimeDiffInSec) > maxTimeDiffInSec) {
            logger.info("Host " + host.getName() + " has a time difference of "
                  + hostTimeDiffInSec
                  + " seconds and is dropped from placement.");
            outOfSyncHosts.add(host.getName());
         }
      }
      for (String host : outOfSyncHosts) {
         container.removeHost(host);
      }

      logger.info("filter hosts by networks.");
      // filter hosts by networks
      List<com.vmware.bdd.spectypes.VcCluster> usedClusters = clusterSpec.getVcClusters();
      List<String> noNetworkHosts = new ArrayList<String>();
      noNetworkHosts = resMgr.filterHostsByNetwork(clusterSpec.getNetworkNames(), usedClusters);

      for (String host : noNetworkHosts) {
         container.removeHost(host);
      }

      Map<String, List<String>> filteredHosts =new HashMap<String, List<String>>();
      if (!outOfSyncHosts.isEmpty()) filteredHosts.put(PlacementUtil.OUT_OF_SYNC_HOSTS, outOfSyncHosts);
      if (!noNetworkHosts.isEmpty()) {
         filteredHosts.put(PlacementUtil.NO_NETWORKS_HOSTS, noNetworkHosts);
         filteredHosts.put(PlacementUtil.NETWORK_NAMES, clusterSpec.getNetworkNames());
      }

      VcVirtualMachine templateVm = getTemplateVM(clusterSpec.getTemplateName());
      container.setTemplateNode(createBaseNodeFromTemplateVm(templateVm));
      if (clusterSpec.getHostToRackMap() != null
            && clusterSpec.getHostToRackMap().size() != 0) {
         container.addRackMap(clusterSpec.getHostToRackMap());
      }

      logger.info("rack topology file validation.");
      // rack topology file validation
      Set<String> validRacks = new HashSet<String>();
      List<AbstractHost> hosts = container.getAllHosts();
      for (AbstractHost host : hosts) {
         if (container.getRack(host) != null) {
            // this rack is valid as it contains at least one host
            validRacks.add(container.getRack(host));
         }
      }

      for (NodeGroupCreate nodeGroup : clusterSpec.getNodeGroups()) {
         if (nodeGroup.getPlacementPolicies() != null
               && nodeGroup.getPlacementPolicies().getGroupRacks() != null
               && validRacks.size() == 0) {
            throw PlacementException.INVALID_RACK_INFO(clusterSpec.getName(),
                  nodeGroup.getName());
         }
      }

      // for instant clone, we need check if the target hosts are with version 6 or higher
      // if user does not set the clone type in serengeti.properties, the default type
      // should still be FAST clone for hosts with 5.x version
      // for cluster resize, the original cloneType should be used
      if ( null == existedNodes ) {
         checkAndUpdateClusterCloneType(clusterSpec, container);
      }

      logger.info("pre-placement task.");
      //pre-placement task
      String clusterCloneType = clusterSpec.getClusterCloneType();
      chooseClusterCloneService(clusterCloneType).preCalculatePlacements(container, clusterSpec, existedNodes);

      List<BaseNode> baseNodes =
            placementService.getPlacementPlan(container, clusterSpec,
                  existedNodes, filteredHosts);
      for (BaseNode baseNode : baseNodes) {
         baseNode.setNodeAction(Constants.NODE_ACTION_CLONING_VM);
      }

      //Check nodegroup reserved cpu&mem ratio, then set the node's cpuReservationMHz = vcpuNum * ESXcpuMhz*ratio
      for (BaseNode current : baseNodes) {
         Float cpuRatio = current.getNodeGroup().getReservedCpuRatio();
         Float memRatio = current.getNodeGroup().getReservedMemRatio();
         //if ReservedCpu_ratio is set, caculate the reservedCpu for vm
         if ( cpuRatio != null  && cpuRatio > 0 && cpuRatio <= 1) {
            if (current.getTargetHost() != null) {
               VcHost host = VcResourceUtils.findHost(current.getTargetHost());
               long nodeCpuMhz =
                     host.getCpuHz() / (1024 * 1024) * current.getCpu();
               current.getVmSchema().resourceSchema.cpuReservationMHz
                     = (long) Math.ceil(nodeCpuMhz * cpuRatio);
               logger.info("Base Node's cpuReservationMHz is set to "
                     + current.getVmSchema().resourceSchema.cpuReservationMHz);
            }
         }
         //Check latency&ReservedMem_ratio is set, caculate the reservedMem for vm
         if ( memRatio != null  && memRatio > 0 && memRatio <= 1) {
            if(current.getVmSchema().resourceSchema.latencySensitivity == LatencyPriority.HIGH)
               current.getVmSchema().resourceSchema.memReservationSize = current.getMem();
            else{
               current.getVmSchema().resourceSchema.memReservationSize
                     = (long) Math.ceil(current.getMem() * memRatio);
            }
            logger.info("Base Node's memReservationM is set to "
                  + current.getVmSchema().resourceSchema.memReservationSize);
         }
      }

      logger.info("Finished calculating provision plan");

      return baseNodes;
   }

   @Override
   public boolean startCluster(final String name,
         List<NodeOperationStatus> failedNodes, StatusUpdater statusUpdator) {
      logger.info("startCluster, start.");
      boolean isMapDistro = clusterEntityMgr.findByName(name).getDistroVendor().equalsIgnoreCase(Constants.MAPR_VENDOR);
      List<NodeEntity> nodes = clusterEntityMgr.findAllNodes(name);
      logger.info("startCluster, start to create store procedures.");
      List<Callable<Void>> storeProcedures = new ArrayList<Callable<Void>>();

      Map<String, NodeOperationStatus> nodesStatus =
            new HashMap<String, NodeOperationStatus>();
      for (int i = 0; i < nodes.size(); i++) {
         NodeEntity node = nodes.get(i);
         VcVirtualMachine vcVm = ClusterUtil.getVcVm(clusterEntityMgr, node);
         if (vcVm == null) {
            // cannot find VM
            logger.info("VC vm does not exist for node: " + node.getVmName());
            continue;
         }

         StartVmSP.StartVmPrePowerOn prePowerOn = new StartVmSP.StartVmPrePowerOn(isMapDistro,
               (new Gson()).toJson(node.getVolumns()));
         StartVmPostPowerOn query =
               new StartVmPostPowerOn(node.fetchAllPortGroups(),
                     Configuration.getInt(Constants.VM_POWER_ON_WAITING_SEC_KEY, Constants.VM_POWER_ON_WAITING_SEC), clusterEntityMgr);
         VcHost host = null;
         if (node.getHostName() != null) {
            host = VcResourceUtils.findHost(node.getHostName());
         }
         StartVmSP startSp = new StartVmSP(vcVm, prePowerOn, query, host);
         storeProcedures.add(startSp);
         nodesStatus.put(node.getVmName(),
               new NodeOperationStatus(node.getVmName()));
      }

      try {
         if (storeProcedures.isEmpty()) {
            logger.info("no VM is available. Return directly.");
            return true;
         }
         Callable<Void>[] storeProceduresArray =
               storeProcedures.toArray(new Callable[0]);
         // execute store procedures to start VMs
         logger.info("ClusteringService, start to start vms.");
         BaseProgressCallback callback = new BaseProgressCallback(statusUpdator);
         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storeProceduresArray, callback);
         if (result == null) {
            for (NodeOperationStatus status : nodesStatus.values()) {
               status.setSucceed(false);
            }
            logger.error("No VM is started.");
            failedNodes.addAll(nodesStatus.values());
            return false;
         }

         boolean success = true;
         int total = 0;
         for (int i = 0; i < storeProceduresArray.length; i++) {
            StartVmSP sp = (StartVmSP) storeProceduresArray[i];
            NodeOperationStatus status = nodesStatus.get(sp.getVmName());
            VcVirtualMachine vm = sp.getVcVm();
            if (result[i].finished && result[i].throwable == null) {
               ++total;
               nodesStatus.remove(status.getNodeName()); // do not return success node
            } else if (result[i].throwable != null) {
               status.setSucceed(false);
               status.setErrorMessage(getErrorMessage(result[i].throwable));
               if (vm != null
                     && vm.isPoweredOn() && VcVmUtil.checkIpAddresses(vm)) {
                  ++total;
                  nodesStatus.remove(status.getNodeName()); // do not return success node status
               } else {
                  if (!vm.isConnected()
                        || vm.getHost().isUnavailbleForManagement()) {
                     logger.error("Cannot start VM " + vm.getName()
                           + " in connection state " + vm.getConnectionState()
                           + " or in maintenance mode. "
                           + "Ignore this VM and continue cluster operations.");
                     continue;
                  }
                  logger.error(
                        "Failed to start VM " + nodes.get(i).getVmName(),
                        result[i].throwable);
                  success = false;
               }
            }
         }
         logger.info(total + " VMs are started.");
         failedNodes.addAll(nodesStatus.values());
         return success;
      } catch (InterruptedException e) {
         logger.error("error in staring VMs", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   @Override
   public boolean stopCluster(final String name,
         List<NodeOperationStatus> failedNodes, StatusUpdater statusUpdator) {
      logger.info("stopCluster, start.");
      List<NodeEntity> nodes = clusterEntityMgr.findAllNodes(name);
      logger.info("stopCluster, start to create store procedures.");
      List<Callable<Void>> storeProcedures = new ArrayList<Callable<Void>>();
      Map<String, NodeOperationStatus> nodesStatus =
            new HashMap<String, NodeOperationStatus>();
      for (int i = 0; i < nodes.size(); i++) {
         NodeEntity node = nodes.get(i);
         VcVirtualMachine vcVm = ClusterUtil.getVcVm(clusterEntityMgr, node);

         if (vcVm == null) {
            logger.info("VC vm does not exist for node: " + node.getVmName());
            continue;
         }
         StopVmSP stopSp = new StopVmSP(vcVm);
         storeProcedures.add(stopSp);
         nodesStatus.put(node.getVmName(),
               new NodeOperationStatus(node.getVmName()));
      }

      try {
         if (storeProcedures.isEmpty()) {
            logger.info("no VM is available. Return directly.");
            return true;
         }
         Callable<Void>[] storeProceduresArray = storeProcedures.toArray(new Callable[0]);
         // execute store procedures to start VMs
         logger.info("ClusteringService, start to stop vms.");
         BaseProgressCallback callback = new BaseProgressCallback(statusUpdator);

         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storeProceduresArray, callback);
         if (result == null) {
            logger.error("No VM is stoped.");
            for (NodeOperationStatus status : nodesStatus.values()) {
               status.setSucceed(false);
            }
            failedNodes.addAll(nodesStatus.values());
            return false;
         }

         boolean success = true;
         int total = 0;
         for (int i = 0; i < storeProceduresArray.length; i++) {
            StopVmSP sp = (StopVmSP) storeProceduresArray[i];
            NodeOperationStatus status = nodesStatus.get(sp.getVmName());
            if (result[i].finished && result[i].throwable == null) {
               ++total;
               nodesStatus.remove(status.getNodeName()); // do not return success node
            } else if (result[i].throwable != null) {
               status.setSucceed(false);
               status.setErrorMessage(getErrorMessage(result[i].throwable));
               VcVirtualMachine vm = sp.getVcVm();
               if (vm == null || vm.isPoweredOff()) {
                  ++total;
                  nodesStatus.remove(status.getNodeName()); // do not return success node
               } else {
                  if (!vm.isConnected()
                        || vm.getHost().isUnavailbleForManagement()) {
                     logger.error("Cannot stop VM " + vm.getName()
                           + " in connection state " + vm.getConnectionState()
                           + " or in maintenance mode. "
                           + "Ignore this VM and continue cluster operations.");
                     continue;
                  }
                  logger.error("Failed to stop VM " + nodes.get(i).getVmName(),
                        result[i].throwable);
                  success = false;
               }
            }
         }
         logger.info(total + " VMs are stoped.");
         return success;
      } catch (InterruptedException e) {
         logger.error("error in stoping VMs", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   private String getErrorMessage(Throwable throwable) {
      if (throwable == null) {
         return null;
      }
      return CommonUtil.getCurrentTimestamp() + " " + throwable.getMessage();
   }

   @Override
   public boolean removeBadNodes(ClusterCreate cluster,
         List<BaseNode> existingNodes, List<BaseNode> deletedNodes,
         Map<String, Set<String>> occupiedIpSets, StatusUpdater statusUpdator) {
      logger.info("Start to remove node violate placement policy "
            + "or in wrong status in cluster: " + cluster.getName());
      // call tm to remove bad nodes
      List<BaseNode> badNodes =
            placementService.getBadNodes(cluster, existingNodes);
      if (badNodes == null) {
         badNodes = new ArrayList<BaseNode>();
      }

      logger.info("violate placement policy nodes: " + badNodes);
      // append node in wrong status
      for (BaseNode node : deletedNodes) {
         if (node.getVmMobId() != null) {
            badNodes.add(node);
         } else {
            node.setSuccess(true);
         }
      }

      if (badNodes != null && badNodes.size() > 0) {
         boolean deleted = syncDeleteVMs(badNodes, statusUpdator, false);
         afterBadVcVmDelete(existingNodes, deletedNodes, badNodes,
               occupiedIpSets);
         return deleted;
      }
      return true;
   }

   @Override
   public List<BaseNode> getBadNodes(ClusterCreate cluster,
         List<BaseNode> existingNodes) {
      return placementService.getBadNodes(cluster, existingNodes);
   }

   private void afterBadVcVmDelete(List<BaseNode> existingNodes,
         List<BaseNode> deletedNodes, List<BaseNode> vcDeletedNodes,
         Map<String, Set<String>> occupiedIpSets) {
      // clean up in memory node list
      deletedNodes.addAll(vcDeletedNodes);
      Set<String> deletedNodeNames = new HashSet<String>();
      for (BaseNode vNode : vcDeletedNodes) {
         deletedNodeNames.add(vNode.getVmName());
      }
      for (Iterator<BaseNode> ite = existingNodes.iterator(); ite.hasNext();) {
         BaseNode vNode = ite.next();
         if (deletedNodeNames.contains(vNode.getVmName())) {
            JobUtils.adjustOccupiedIpSets(occupiedIpSets, vNode, false);
            ite.remove();
         }
      }
   }

   @Override
   public boolean deleteCluster(String name, List<BaseNode> vNodes, StatusUpdater statusUpdator) {
      if (vNodes == null || vNodes.size() < 1) {
         logger.error("no node need to be deleted.");
         return true;
      }
      //Find folder firstly, or will fail to find it after deleting VMs.
      Folder folder = findFolder(vNodes.get(0));
      boolean deleted = syncDeleteVMs(vNodes, statusUpdator, true);
      try {
         deleteChildRps(name, vNodes);
      } catch (Exception e) {
         logger.error("ignore delete resource pool error.", e);
      }
      try {
         deleteFolders(folder);
      } catch (Exception e) {
         logger.error("ignore delete folder error.", e);
      }
      return deleted;
   }

   private void deleteChildRps(String hadoopClusterName, List<BaseNode> vNodes) {
      logger.info("Start to delete child resource pools for cluster: "
              + hadoopClusterName);
      Map<String, Map<String, VcResourcePool>> clusterMap =
            new HashMap<String, Map<String, VcResourcePool>>();
      for (BaseNode node : vNodes) {
         String vcClusterName = node.getTargetVcCluster();
         AuAssert.check(vcClusterName != null);
         String vcRpName = node.getTargetRp();
         if (clusterMap.get(vcClusterName) == null) {
            clusterMap
                  .put(vcClusterName, new HashMap<String, VcResourcePool>());
         }
         Map<String, VcResourcePool> rpMap = clusterMap.get(vcClusterName);
         if (rpMap.get(vcRpName) == null) {
            VcResourcePool vcRp =
                  VcResourceUtils.findRPInVCCluster(vcClusterName, vcRpName);
            if (vcRp != null) {
               rpMap.put(vcRpName, vcRp);
            }
         }
      }
      List<VcResourcePool> rps = new ArrayList<VcResourcePool>();
      for (Map<String, VcResourcePool> map : clusterMap.values()) {
         rps.addAll(map.values());
      }
      Callable<Void>[] storedProcedures = new Callable[rps.size()];
      String childRp = ConfigInfo.getSerengetiUUID() + "-" + hadoopClusterName;
      int i = 0;
      for (VcResourcePool rp : rps) {
         DeleteRpSp sp = new DeleteRpSp(rp, childRp);
         storedProcedures[i] = sp;
         i++;
      }
      try {
         NoProgressUpdateCallback callback = new NoProgressUpdateCallback();
         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storedProcedures, callback);
         if (result == null || result.length == 0) {
            logger.error("No rp is deleted.");
            return;
         }
         int total = 0;
         for (int j = 0; j < storedProcedures.length; j++) {
            if (result[j].throwable != null) {
               DeleteRpSp sp = (DeleteRpSp) storedProcedures[j];
               logger.error(
                     "Failed to delete child resource pool "
                           + sp.getDeleteRpName() + " under " + sp.getVcRp(),
                     result[j].throwable);
            } else {
               total++;
            }
         }
      } catch (InterruptedException e) {
         logger.error("error in deleting resource pools", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   private Folder findFolder(BaseNode node) throws BddException {
      String path = node.getVmFolder();
      String mobid = node.getVmMobId();
      // path format: <serengeti...>/<cluster name>/<group name>
      String[] folderNames = path.split("/");
      AuAssert.check(folderNames.length == 3);
      VcDatacenter dc = VcResourceUtils.findVM(mobid).getDatacenter();
      List<String> deletedFolders = new ArrayList<String>();
      deletedFolders.add(folderNames[0]);
      deletedFolders.add(folderNames[1]);
      Folder folder = null;
      try {
         folder = VcResourceUtils.findFolderByNameList(dc, deletedFolders);
      } catch (Exception e) {
         logger.error("error in finding folders", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
      String clusterFolderName = folderNames[0] + "/" + folderNames[1];
      logger.info("find cluster root folder: " + clusterFolderName + "test: folder.name=" + folder.getName() + " folder.ref=" + folder._getRef().getValue());
      return folder;
   }
   
   /**
    * this method will delete the cluster root folder, if there is any VM
    * existed and powered on in the folder, the folder deletion will fail.
    *
    * @param folder
    * @throws BddException
    */
   private void deleteFolders(Folder folder) throws BddException {
      if (folder == null) {
         logger.info("No folder to delete.");
         return;
      }
      List<Folder> folders = new ArrayList<Folder>();
      folders.add(folder);
      DeleteVMFolderSP sp = new DeleteVMFolderSP(folders, true, false);
      Callable<Void>[] storedProcedures = new Callable[1];
      storedProcedures[0] = sp;
      try {
         NoProgressUpdateCallback callback = new NoProgressUpdateCallback();
         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storedProcedures, callback);
         if (result == null || result.length == 0) {
            logger.error("No folder is deleted.");
            return;
         }
         if (result[0].finished && result[0].throwable == null) {
            logger.info("Cluster folder " + folder.getName() + " is deleted.");
         } else {
            logger.info("Failed to delete cluster folder " + folder.getName(),
                  result[0].throwable);
         }
      } catch (InterruptedException e) {
         logger.error("error in deleting folders", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   @Override
   @SuppressWarnings("unchecked")
   public boolean syncDeleteVMs(List<BaseNode> badNodes,
         StatusUpdater statusUpdator, boolean ignoreUnavailableNodes) {
      logger.info("syncDeleteVMs, start to create store procedures.");
      List<Callable<Void>> storeProcedures = new ArrayList<Callable<Void>>();
      List<BaseNode> toBeDeleted = new ArrayList<BaseNode>();
      for (int i = 0; i < badNodes.size(); i++) {
         BaseNode node = badNodes.get(i);
         if (node.getVmMobId() == null) {
            // vm is already deleted
            node.setSuccess(true);
            continue;
         }
         DeleteVmByIdSP deleteSp = new DeleteVmByIdSP(node.getVmMobId());
         storeProcedures.add(deleteSp);
         toBeDeleted.add(node);
      }
      try {
         if (storeProcedures.isEmpty()) {
            logger.info("no VM is created. Return directly.");
            return true;
         }
         Callable<Void>[] storeProceduresArray =
               storeProcedures.toArray(new Callable[0]);
         // execute store procedures to delete VMs
         logger.info("ClusteringService, start to delete vms.");
         BaseProgressCallback callback =
               new BaseProgressCallback(statusUpdator, 0, 50);
         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storeProceduresArray, callback);
         if (result == null) {
            logger.error("No VM is deleted.");
            return false;
         }

         int total = 0;
         boolean failed = false;
         for (int i = 0; i < storeProceduresArray.length; i++) {
            BaseNode vNode = toBeDeleted.get(i);
            vNode.setFinished(true);
            if (result[i].finished && result[i].throwable == null) {
               vNode.setSuccess(true);
               vNode.setVmMobId(null);
               ++total;
            } else if (result[i].throwable != null) {
               vNode.setSuccess(false);
               vNode.setErrMessage(getErrorMessage(result[i].throwable));
               if (ignoreUnavailableNodes) {
                  DeleteVmByIdSP sp = (DeleteVmByIdSP) storeProceduresArray[i];
                  VcVirtualMachine vcVm = sp.getVcVm();
                  if (!vcVm.isConnected()
                        || vcVm.getHost().isUnavailbleForManagement()) {
                     logger.error("Failed to delete VM " + vcVm.getName()
                           + " in connection state "
                           + vcVm.getConnectionState()
                           + "  or in maintenance mode.");
                     logger.error("Ignore this failure and continue cluster operations.");
                     continue;
                  }
               }
               logger.error("Failed to delete VM " + vNode.getVmName(),
                     result[i].throwable);
               failed = true;
            }
            vNode.setFinished(true);
         }
         logger.info(total + " VMs are deleted.");
         return !failed;
      } catch (InterruptedException e) {
         logger.error("error in deleting VMs", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   @Override
   public UUID reserveResource(String clusterName) {
      ResourceReservation reservation = new ResourceReservation();
      reservation.setClusterName(clusterName);
      return resMgr.reserveResource(reservation);
   }

   @Override
   public void commitReservation(UUID reservationId) throws VcProviderException {
      resMgr.commitReservation(reservationId);
   }

   @Override
   public Map<String, String> configIOShares(String clusterName, List<NodeEntity> targetNodes,
         Priority ioShares) {
      AuAssert.check(clusterName != null && targetNodes != null
              && !targetNodes.isEmpty());

      Callable<Void>[] storeProcedures = new Callable[targetNodes.size()];
      int i = 0;
      for (NodeEntity node : targetNodes) {
         ConfigIOShareSP ioShareSP =
               new ConfigIOShareSP(node.getMoId(), ioShares);
         storeProcedures[i] = ioShareSP;
         i++;
      }

      try {
         // execute store procedures to configure io shares
         logger.info("ClusteringService, start to reconfigure vm's io shares.");
         NoProgressUpdateCallback callback = new NoProgressUpdateCallback();
         ExecutionResult[] result =
               Scheduler
                     .executeStoredProcedures(
                           com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                           storeProcedures, callback);
         if (result == null) {
            logger.error("No VM's io share level is reconfigured.");
            throw ClusteringServiceException
                  .RECONFIGURE_IO_SHARE_FAILED(clusterName);
         }

         int total = 0;
         Map<String, String> failedNodes = new HashMap<String, String>();
         for (i = 0; i < storeProcedures.length; i++) {
            if (result[i].finished && result[i].throwable == null) {
               ++total;
            } else if (result[i].throwable != null) {
               logger.error("Failed to reconfigure vm", result[i].throwable);
               String nodeName = targetNodes.get(i).getVmName();
               String message = CommonUtil.getCurrentTimestamp() + " " + result[i].throwable.getMessage();
               failedNodes.put(nodeName, message);
            }
         }
         logger.info(total + " vms are reconfigured.");
         return failedNodes;
      } catch (InterruptedException e) {
         logger.error("error in reconfiguring vm io shares", e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   @Override
   public boolean startSingleVM(String clusterName, String nodeName,
         StatusUpdater statusUpdator) {
      NodeEntity node = this.clusterEntityMgr.findNodeByName(nodeName);
      boolean reserveRawDisks = clusterEntityMgr.findByName(clusterName).getDistroVendor().equalsIgnoreCase(Constants.MAPR_VENDOR);
      // For node scale up/down, the disk info in db is not yet updated when powering on it, need to fetch from VC
      List<DiskSpec> diskSpecList = VcVmUtil.toDiskSpecList(node.getDisks());

      StartVmSP.StartVmPrePowerOn prePowerOn = new StartVmSP.StartVmPrePowerOn(reserveRawDisks,
            VcVmUtil.getVolumesFromSpecs(node.getMoId(), diskSpecList));
      StartVmPostPowerOn query =
            new StartVmPostPowerOn(node.fetchAllPortGroups(),
                  Configuration.getInt(Constants.VM_POWER_ON_WAITING_SEC_KEY, Constants.VM_POWER_ON_WAITING_SEC), clusterEntityMgr);

      VcHost host = null;
      if (node.getHostName() != null) {
         host = VcResourceUtils.findHost(node.getHostName());
      }

      VcVirtualMachine vcVm = ClusterUtil.getVcVm(clusterEntityMgr, node);
      if (vcVm == null) {
         logger.info("VC vm does not exist for node: " + node.getVmName());
         return false;
      }
      StartVmSP startVMSP = new StartVmSP(vcVm, prePowerOn, query, host);
      return VcVmUtil.runSPOnSingleVM(node, startVMSP);
   }

   @Override
   public boolean stopSingleVM(String clusterName, String nodeName,
         StatusUpdater statusUpdator, boolean... vmPoweroff) {
      NodeEntity node = this.clusterEntityMgr.findNodeByName(nodeName);
      VcVirtualMachine vcVm = ClusterUtil.getVcVm(clusterEntityMgr, node);
      if (vcVm == null) {
         // cannot find VM
         logger.info("VC vm does not exist for node: " + node.getVmName());
         return false;
      }
      StopVmSP stopVMSP;
      if (vmPoweroff.length > 0 && vmPoweroff[0]) {
         stopVMSP = new StopVmSP(vcVm, true);
      } else {
         stopVMSP = new StopVmSP(vcVm);
      }
      return VcVmUtil.runSPOnSingleVM(node, stopVMSP);
   }

   @Override
   public VmEventManager getEventProcessor() {
      return this.processor;
   }

   @Override
   public boolean isSupportVHM(String clusterName) {
      ClusterEntity cluster = getClusterEntityMgr().findByName(clusterName);
      SoftwareManager softMgr =
            softwareManagerCollector
                  .getSoftwareManagerByClusterName(clusterName);
      if (!softMgr.hasComputeMasterGroup(getClusterEntityMgr()
            .toClusterBluePrint(clusterName))) {
         logger.warn("Use of auto elasticity, must configure Jobtracker or ResourceManager.");
         return false;
      }
      return true;
   }

   @Override
   public boolean addNodeGroups(ClusterCreate clusterSpec, NodeGroupCreate[] nodeGroupsAdd,
                               List<BaseNode> vNodes) {
      boolean success = false;
      List<NodeGroupCreate> newNodeGroups = new ArrayList<NodeGroupCreate>();

      if (clusterSpec != null && clusterSpec.getNodeGroups() != null) {
         for (NodeGroupCreate ng : clusterSpec.getNodeGroups()) {
            newNodeGroups.add(ng);
         }
      }

      if (nodeGroupsAdd != null) {
         for (NodeGroupCreate ng : nodeGroupsAdd) {
            newNodeGroups.add(ng);
         }
      }

      if (clusterSpec != null) {
         clusterSpec.setNodeGroups(newNodeGroups
                 .toArray(new NodeGroupCreate[newNodeGroups.size()]));
      }

      if(null != createVcFolders(clusterSpec, true)) {
         if(null != createVcResourcePools(vNodes, true)) {
            success = true;
         }
      }

      return success;
   }

   private void checkAndUpdateClusterCloneType(ClusterCreate clusterSpec, Container container) {
      String clusterName = clusterSpec.getName();
      String type = Configuration.getString("cluster.clone.service");
      if ( StringUtils.isBlank(type) ) {
         // check if there are any ESXi hosts with version lower than 6.0
         boolean isLowVersion = false;
         for ( AbstractHost host : container.getAllHosts() ) {
            String hostName = host.getName();
            final VcHost vchost = VcResourceUtils.findHost(hostName);
            AuAssert.check(vchost != null, String.format("can' find host: %1s in VC.", hostName));
            String version = vchost.getVersion();
            if ( Version.compare(version, Constants.VCENTER_VERSION_6 ) < 0 ) {
               logger.info("Some ESXi host version is lower than 6.0.");
               isLowVersion = true;
               break;
            }
         }

         // if some ESXi host version is lower than 6.0, update the cluster clone type to FAST
         if ( isLowVersion ) {
            ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
            String advProps = cluster.getAdvancedProperties();
            if (!CommonUtil.isBlank(advProps)) {
               Gson gson = new Gson();
               Map<String, String> advancedProperties =
                     gson.fromJson(advProps, Map.class);
               String cloneType = advancedProperties.get("ClusterCloneType");
               if ( !StringUtils.isBlank(cloneType)
                     && cloneType.equals(Constants.CLUSTER_CLONE_TYPE_INSTANT_CLONE) ) {
                  cloneType = Constants.CLUSTER_CLONE_TYPE_FAST_CLONE;
                  advancedProperties.put("ClusterCloneType", cloneType);
                  cluster.setAdvancedProperties(gson.toJson(advancedProperties));
                  logger.info("Change cluster clone type to fast clone because of ESXi version lower than 6.0.");
                  clusterEntityMgr.update(cluster);
                  clusterSpec.setClusterCloneType(cloneType);
               }
            }
         }
      }
   }

}