ClusterHealService.java

/***************************************************************************
 * Copyright (c) 2012-2015 VMware, Inc. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ***************************************************************************/
package com.vmware.bdd.service.impl;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;

import com.google.gson.internal.Pair;

import com.vmware.aurora.global.Configuration;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.google.gson.internal.Pair;
import com.vmware.aurora.composition.CreateVmSP;
import com.vmware.aurora.composition.VmSchema;
import com.vmware.aurora.composition.compensation.CompensateCreateVmSP;
import com.vmware.aurora.composition.concurrent.ExecutionResult;
import com.vmware.aurora.composition.concurrent.Scheduler;
import com.vmware.aurora.vc.DeviceId;
import com.vmware.aurora.vc.VcCache;
import com.vmware.aurora.vc.VcDatacenter;
import com.vmware.aurora.vc.VcDatastore;
import com.vmware.aurora.vc.VcHost;
import com.vmware.aurora.vc.VcResourcePool;
import com.vmware.aurora.vc.VcVirtualMachine;
import com.vmware.aurora.vc.VcVmCloneType;
import com.vmware.aurora.vc.VmConfigUtil;
import com.vmware.aurora.vc.vcservice.VcContext;
import com.vmware.aurora.vc.vcservice.VcSession;
import com.vmware.bdd.apitypes.ClusterCreate;
import com.vmware.bdd.apitypes.NetworkAdd;
import com.vmware.bdd.apitypes.NodeGroupCreate;
import com.vmware.bdd.apitypes.NodeStatus;
import com.vmware.bdd.apitypes.StorageRead.DiskType;
import com.vmware.bdd.entity.DiskEntity;
import com.vmware.bdd.entity.NodeEntity;
import com.vmware.bdd.exception.BddException;
import com.vmware.bdd.exception.ClusterHealServiceException;
import com.vmware.bdd.exception.ClusteringServiceException;
import com.vmware.bdd.manager.ClusterConfigManager;
import com.vmware.bdd.manager.intf.IClusterEntityManager;
import com.vmware.bdd.placement.entity.AbstractDatacenter.AbstractDatastore;
import com.vmware.bdd.service.IClusterHealService;
import com.vmware.bdd.service.IClusteringService;
import com.vmware.bdd.service.resmgmt.INetworkService;
import com.vmware.bdd.service.sp.NoProgressUpdateCallback;
import com.vmware.bdd.service.sp.ReplaceVmBadDisksSP;
import com.vmware.bdd.service.sp.ReplaceVmPrePowerOn;
import com.vmware.bdd.service.sp.StartVmPostPowerOn;
import com.vmware.bdd.service.sp.StartVmSP;
import com.vmware.bdd.service.utils.VcResourceUtils;
import com.vmware.bdd.specpolicy.GuestMachineIdSpec;
import com.vmware.bdd.spectypes.DiskSpec;
import com.vmware.bdd.utils.AuAssert;
import com.vmware.bdd.utils.ClusterUtil;
import com.vmware.bdd.utils.Constants;
import com.vmware.bdd.utils.JobUtils;
import com.vmware.bdd.utils.VcVmUtil;
import com.vmware.vim.binding.vim.Folder;
import com.vmware.vim.binding.vim.vm.device.VirtualDisk;

@Service
public class ClusterHealService implements IClusterHealService {
   private static final Logger logger = Logger
         .getLogger(ClusterHealService.class);

   private static final String RECOVERY_VM_NAME_POSTFIX = "-recovery";

   private IClusterEntityManager clusterEntityMgr;

   private ClusterConfigManager configMgr;

   private IClusteringService clusteringService;

   private INetworkService networkMgr;

   private final String IMAGESTORE = "IMAGESTORE";

   private final String DISKSTORE = "DISKSTORE";

   public IClusterEntityManager getClusterEntityMgr() {
      return clusterEntityMgr;
   }

   @Autowired
   public void setClusterEntityMgr(IClusterEntityManager clusterEntityMgr) {
      this.clusterEntityMgr = clusterEntityMgr;
   }

   public ClusterConfigManager getConfigMgr() {
      return configMgr;
   }

   @Autowired
   public void setConfigMgr(ClusterConfigManager configMgr) {
      this.configMgr = configMgr;
   }

   public IClusteringService getClusteringService() {
      return clusteringService;
   }

   @Autowired
   public void setClusteringService(IClusteringService clusteringService) {
      this.clusteringService = clusteringService;
   }

   public INetworkService getNetworkMgr() {
      return networkMgr;
   }

   @Autowired
   public void setNetworkMgr(INetworkService networkMgr) {
      this.networkMgr = networkMgr;
   }

   @Override
   public boolean hasBadDisks(String nodeName) {
      return CollectionUtils.isNotEmpty(getBadDisks(nodeName));
   }

   @Override
   public List<DiskSpec> getBadDisks(String nodeName) {
      return getBadDisksByType(nodeName, null);
   }

   @Override
   public boolean hasBadDisksExceptSystem(String nodeName) {

      if (hasBadSystemDisks(nodeName)) {
         return false;
      }

      boolean hasBadDataDisks = CollectionUtils.isNotEmpty(getBadDataDisks(nodeName));

      boolean hasBadSwapDisks = CollectionUtils.isNotEmpty(getBadSwapDisks(nodeName));

      if (hasBadDataDisks || hasBadSwapDisks) {
         return true;
      }

      return false;
   }

   @Override
   public boolean hasBadSystemDisks(String nodeName) {
      return CollectionUtils.isNotEmpty(getBadSystemDisks(nodeName));
   }

   @Override
   public List<DiskSpec> getBadSystemDisks(String nodeName) {
      return getBadDisksByType(nodeName, DiskType.SYSTEM_DISK);
   }

   @Override
   public List<DiskEntity> getBadSystemDiskEntities(String nodeName) {
      return getBadDiskEntitiesByType(nodeName, DiskType.SYSTEM_DISK);
   }

   @Override
   public boolean hasBadSwapDisks(String nodeName) {
      return CollectionUtils.isNotEmpty(getBadSwapDisks(nodeName));
   }

   @Override
   public List<DiskSpec> getBadSwapDisks(String nodeName) {
      return getBadDisksByType(nodeName, DiskType.SWAP_DISK);
   }

   @Override
   public List<DiskEntity> getBadSwapDiskEntities(String nodeName) {
      return getBadDiskEntitiesByType(nodeName, DiskType.SWAP_DISK);
   }

   @Override
   public boolean hasBadDataDisks(String nodeName) {

      if (hasBadSystemDisks(nodeName)) {
         return false;
      }

      return CollectionUtils.isNotEmpty(getBadDataDisks(nodeName));
   }

   @Override
   public List<DiskSpec> getBadDataDisks(String nodeName) {
      return getBadDisksByType(nodeName, DiskType.DATA_DISK);
   }

   @Override
   public List<DiskEntity> getBadDataDiskEntities(String nodeName) {
      return getBadDiskEntitiesByType(nodeName, DiskType.DATA_DISK);
   }

   private List<DiskSpec> getBadDisksByType(String nodeName, DiskType diskType) {
      List<DiskSpec> badDisks = new ArrayList<>();

      List<DiskEntity> badDiskEntities = getBadDiskEntitiesByType(nodeName, diskType);
      for (DiskEntity badDiskEntity : badDiskEntities) {
         badDisks.add(badDiskEntity.toDiskSpec());
      }

      return badDisks;
   }

   private List<DiskEntity> getBadDiskEntitiesByType(String nodeName, DiskType diskType) {
      List<DiskEntity> diskEntities = clusterEntityMgr.getDisks(nodeName);
      List<DiskEntity> badDiskEntities = new ArrayList<>();

      // scan all disks with disk type and filter out those don't have backing vmdk files or
      // whoes vmdk file attaches to unaccessible datastores
      for (DiskEntity diskEntity : diskEntities) {
         if (diskType != null && !diskEntity.getDiskType().equals(diskType.type)) {
            continue;
         }
         if (isBadDisk(diskEntity)) {
            badDiskEntities.add(diskEntity);
         }
      }

      return badDiskEntities;
   }

   private boolean isBadDisk(DiskEntity disk) {
      boolean isBadDisk = false;

      NodeEntity node = disk.getNodeEntity();
      if (node.isNotExist()) {
         logger.info("The VM " + node.getVmName() + " is not exist.");
         return true;
      }

      if (isBadVmdkPath(disk) || isBadDataStore(disk)) {
         isBadDisk = true;
         logger.info("disk " + disk.getName() + " is bad as datastore " + disk.getDatastoreName() + " is not accessible");
      }

      return isBadDisk;
   }

   private boolean isBadDataStore(DiskEntity disk) {
      return (disk.getDatastoreMoId() != null && !VcVmUtil.isDatastoreAccessible(disk.getDatastoreMoId()));
   }

   private boolean isBadVmdkPath(final DiskEntity disk) {

      if (disk.getVmdkPath() == null || disk.getVmdkPath().isEmpty()) {
         return true;
      }

      boolean  isBadVmdkPath = VcContext.inVcSessionDo(new VcSession<Boolean>() {

         @Override
         protected Boolean body() throws Exception {

            DeviceId deviceId = disk.getDiskDeviceId();
            if (deviceId == null) {
               return true;
            }

            VcVirtualMachine vm = VcCache.getIgnoreMissing(disk.getNodeEntity().getMoId());

            VirtualDisk vmdk = (VirtualDisk)vm.getVirtualDevice(deviceId);

            if (vmdk == null || !disk.getVmdkPath().equals(VmConfigUtil.getVmdkPath(vmdk))) {
               return true;
            }

            return false;
         }

         @Override
         protected boolean isTaskSession() {
            return true;
         }

      });

      return isBadVmdkPath;
   }

   private Map<String, List<VcDatastore>> filterDatastores(VcHost targetHost, ClusterCreate spec, String groupName, NodeEntity nodeEntity) {

      Map<String, List<VcDatastore>> candidates = new HashMap<String, List<VcDatastore>>();

      candidates.put(IMAGESTORE, filterDatastoresWithPattern(targetHost, spec, groupName, nodeEntity, true));

      candidates.put(DISKSTORE, filterDatastoresWithPattern(targetHost, spec, groupName, nodeEntity, false));

      return candidates;
   }

   private List<VcDatastore> filterDatastoresWithPattern(VcHost targetHost, ClusterCreate spec, String groupName, NodeEntity nodeEntity, boolean isImagestoreNamePattern) {

      NodeGroupCreate groupSpec = spec.getNodeGroup(groupName);

      String[] datastoreNamePatterns = null;
      if (isImagestoreNamePattern) {
         datastoreNamePatterns = NodeGroupCreate.getImagestoreNamePattern(spec, groupSpec);
      } else {
         datastoreNamePatterns = NodeGroupCreate.getDiskstoreNamePattern(spec, groupSpec);
      }

      List<VcDatastore> candidates = new ArrayList<VcDatastore>();
      for (VcDatastore ds : targetHost.getDatastores()) {
         if (!ds.isAccessible()) {
            continue;
         }
         for (String pattern : datastoreNamePatterns) {
            if (ds.getName().matches(pattern)) {
               candidates.add(ds);
               break;
            }
         }
      }

      return candidates;
   }

   private AbstractDatastore getLeastUsedDatastore(
         Map<AbstractDatastore, Integer> usage, int requiredInGB) {
      int min = Integer.MAX_VALUE;
      AbstractDatastore result = null;

      for (AbstractDatastore ads : usage.keySet()) {
         if (ads.getFreeSpace() >= requiredInGB && usage.get(ads) < min) {
            min = usage.get(ads);
            result = ads;
         }
      }

      return result;
   }

   private void findReplacementDisks(String nodeName,
         List<DiskSpec> badDisks, Map<AbstractDatastore, Integer> usage) {
      // reverse sort, in descending order
      // bin pack problem, place large disk first.
      Collections.sort(badDisks);

      for (DiskSpec disk : badDisks) {
         int requiredSize = disk.getSize();
         AbstractDatastore ads = getLeastUsedDatastore(usage, requiredSize);
         if (ads == null) {
            throw ClusterHealServiceException.NOT_ENOUGH_STORAGE(nodeName,
                  "Cannot find a datastore with enough space to place disk "
                        + disk.getName() + " of size " + disk.getSize()
                        + " GB");
         }

         disk.setTargetDs(ads.getName());
         disk.setVmdkPath(null);

         // deduct space
         ads.allocate(requiredSize);
         // increase reference by 1
         usage.put(ads, usage.get(ads) + 1);
      }
   }

   @Override
   public List<DiskSpec> getReplacementDisks(String clusterName, String groupName, String nodeName, List<DiskSpec> badDisks) {

      ClusterCreate spec = configMgr.getClusterConfig(clusterName);

      NodeEntity nodeEntity = clusterEntityMgr.findByName(clusterName, groupName, nodeName);

      VcHost targetHost = getTargetHost(nodeEntity);

      List<DiskSpec> replacementDisks = new ArrayList<DiskSpec>();

      Map<String, List<VcDatastore>> validDatastores = filterDatastores(targetHost, spec, groupName, nodeEntity);

      List<DiskSpec> badSystemDisks = new ArrayList<DiskSpec>();
      for (DiskSpec badDisk : badDisks) {
         if (badDisk.isSystemDisk() || badDisk.isSwapDisk()) {
            badSystemDisks.add(badDisk);
         }
      }
      if (!badSystemDisks.isEmpty()) {
         List<DiskSpec> replacementSystemDisks = replacementDisks(clusterName, groupName, nodeName, badSystemDisks, targetHost, validDatastores.get(IMAGESTORE));
         replacementDisks.addAll(replacementSystemDisks);
      }

      List<DiskSpec> badDataDisks = new ArrayList<DiskSpec>();
      for (DiskSpec badDisk : badDisks) {
         if (badDisk.isDataDisk()) {
            badDataDisks.add(badDisk);
         }
      }
      if (!badDataDisks.isEmpty()) {
         List<DiskSpec> replacementDataDisks = replacementDisks(clusterName, groupName, nodeName, badDataDisks, targetHost, validDatastores.get(DISKSTORE));
         replacementDisks.addAll(replacementDataDisks);
      }

      return replacementDisks;
   }

   private List<DiskSpec> replacementDisks(String clusterName, String groupName, String nodeName, List<DiskSpec> badDisks, VcHost targetHost, List<VcDatastore> validDatastores) {

      // initialize env for placement algorithm
      int totalSizeInGB = 0;
      Map<AbstractDatastore, Integer> usage = new HashMap<AbstractDatastore, Integer>(validDatastores.size());
      List<AbstractDatastore> pools = new ArrayList<AbstractDatastore>(validDatastores.size());

      for (VcDatastore ds : validDatastores) {
         totalSizeInGB += ds.getFreeSpace() >> 30;
         AbstractDatastore ads =
               new AbstractDatastore(ds.getName(),
                     (int) (ds.getFreeSpace() >> 30));
         pools.add(ads);
         usage.put(ads, 0);
      }

      int requiredSizeInGB = 0;
      for (DiskSpec disk : badDisks) {
         requiredSizeInGB += disk.getSize();
      }

      if (totalSizeInGB < requiredSizeInGB) {
         throw ClusterHealServiceException.NOT_ENOUGH_STORAGE(nodeName,
               "" + requiredSizeInGB + " GB storage is required on host "
                     + targetHost.getName() + ", but only " + totalSizeInGB
                     + " GB available");
      }

      List<DiskEntity> goodDisks = clusterEntityMgr.getDisks(nodeName);

      // collects datastore usages
      for (DiskEntity disk : goodDisks) {
         boolean bad = false;
         for (DiskSpec diskSpec : badDisks) {
            if (StringUtils.equals(disk.getName(), diskSpec.getName())) {
               bad = true;
               break;
            }
         }

         // ignore bad disks
         if (bad)
            continue;

         AbstractDatastore targetAds = null;
         for (AbstractDatastore ads : pools) {
            if (ads.getName().equals(disk.getDatastoreName())) {
               targetAds = ads;
               break;
            }
         }

         if (targetAds == null)
            logger.warn("a healthy disk " + disk.getName()
                  + " used a datastore " + disk.getDatastoreName()
                  + "that is not specified in cluster spec");
         else {
            Integer val = usage.get(targetAds);
            usage.put(targetAds, val + 1);
         }
      }

      findReplacementDisks(nodeName, badDisks, usage);

      return badDisks;
   }

   private VcDatastore getTargetDatastore(List<DiskSpec> fullDiskList) {
      String datastore = null;

      for (DiskSpec disk : fullDiskList) {
         if (DiskType.SYSTEM_DISK.equals(disk.getDiskType())) {
            datastore = disk.getTargetDs();
         }
      }
      AuAssert.check(datastore != null);

      VcDatastore ds = VcResourceUtils.findDSInVcByName(datastore);
      if (ds != null) {
         return ds;
      }
      logger.error("target data store " + datastore + " is not found.");
      throw ClusteringServiceException.TARGET_VC_DATASTORE_NOT_FOUND(datastore);
   }

   private Folder getTargetFolder(final NodeEntity node) {
      return VcContext.inVcSessionDo(new VcSession<Folder>() {
         @Override
         protected Folder body() throws Exception {

            if (node.getMoId() != null) {
               VcVirtualMachine vm = VcCache.get(node.getMoId());
               return vm.getParentFolder();
            }

            String folderPath = node.getNodeGroup().getVmFolderPath();
            try {
               List<String> folderNames = Arrays.asList(folderPath.split("/"));
               AuAssert.check(folderNames.size() == 3);

               VcVirtualMachine templateVm = VcCache.get(node.getNodeGroup().getCluster().getTemplateId());
               VcDatacenter dc = templateVm.getDatacenter();

               return VcResourceUtils.findFolderByNameList(dc, folderNames);
            } catch (Exception e) {
               logger.error("error in finding folder " + folderPath , e);
               throw BddException.INTERNAL(e, e.getMessage());
            }

         }
      });
   }

   private VcHost getTargetHost(NodeEntity node) {
      String targetHostName = node.getHostName();

      VcHost targetHost = VcResourceUtils.findHost(targetHostName);

      if (targetHost == null) {
         logger.error("Cannot find the vCenter Server host " + targetHostName + " for node " + node.getVmName());
         throw ClusterHealServiceException.TARGET_VC_HOST_NOT_FOUND(targetHostName, node.getVmName());
      }

      return targetHost;
   }

   private CreateVmSP getReplacementVmSp(ClusterCreate clusterSpec,
         String groupName, NodeEntity node, List<DiskSpec> fullDiskSet) {
      VmSchema createSchema =
            VcVmUtil.getVmSchema(clusterSpec, groupName, fullDiskSet,
                  node.getNodeGroup().getCluster().getTemplateId(),
                  Constants.ROOT_SNAPSTHOT_NAME);

      Map<String, String> guestVariable = generateMachineId(clusterSpec, node);

      // TODO: rafactor this function
      VcVmUtil.addBootupUUID(guestVariable);

      boolean ha = getHaFlag(clusterSpec, groupName);

      boolean ft = getFtFlag(clusterSpec, groupName);

      boolean isMapDistro = clusterEntityMgr.findByName(clusterSpec.getName()).getDistroVendor().equalsIgnoreCase(Constants.MAPR_VENDOR);

      // delete old vm and rename new vm in the prePowerOn
      ReplaceVmPrePowerOn prePowerOn =
            new ReplaceVmPrePowerOn(node.getMoId(), node.getVmName(),
                  clusterSpec.getNodeGroup(groupName).getStorage().getShares(),
                  createSchema.networkSchema, createSchema.diskSchema, ha, ft, isMapDistro);

      // power on the new vm, but not wait for ip address here. we have startVmStep to wait for ip
      String newVmName = node.getVmName();
      if (node.getMoId() != null && !node.getMoId().isEmpty()) {
         newVmName = node.getVmName() + RECOVERY_VM_NAME_POSTFIX;
      }
      return new CreateVmSP(newVmName,
            createSchema, VcVmUtil.getTargetRp(clusterSpec.getName(),
                  groupName, node), getTargetDatastore(fullDiskSet),
            prePowerOn, null, guestVariable, VcVmCloneType.FULL, true, getTargetFolder(node),
            getTargetHost(node));
   }

   @Override
   @SuppressWarnings("unchecked")
   public VcVirtualMachine createReplacementVm(String clusterName,
         String groupName, String nodeName, List<DiskSpec> replacementDisks) {
      ClusterCreate spec = configMgr.getClusterConfig(clusterName);
      NodeEntity node = clusterEntityMgr.findByName(spec.getName(), groupName, nodeName);

      List<DiskSpec> fullDiskList = getReplacedFullDisks(node.getVmName(), replacementDisks);

      CreateVmSP cloneVmSp = getReplacementVmSp(spec, groupName, node, fullDiskList);

      CompensateCreateVmSP deleteVmSp = new CompensateCreateVmSP(cloneVmSp);

      Pair<Callable<Void>, Callable<Void>>[] storeProcedures = new Pair[1];
      storeProcedures[0] =
            new Pair<Callable<Void>, Callable<Void>>(cloneVmSp, deleteVmSp);

      // execute store procedures to create VMs
      logger.info("ClusterHealService, start to create replacement vm for node "
            + nodeName);
      Pair<ExecutionResult, ExecutionResult>[] result;
      try {
         result =
               Scheduler
               .executeStoredProcedures(
                     com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                     storeProcedures, 1, null);

         if (result == null) {
            logger.error("vm creation failed for node " + nodeName);
            return null;
         }

         Pair<ExecutionResult, ExecutionResult> pair = result[0];
         CreateVmSP sp = (CreateVmSP) storeProcedures[0].first;
         if (pair.first.finished && pair.first.throwable == null
               && pair.second.finished == false) {
            VcVirtualMachine vm = sp.getVM();
            AuAssert.check(vm != null);
            return vm;
         } else if (pair.first.throwable != null) {
            logger.error(
                  "Failed to create replacement virtual machine for node " + node.getVmName(),
                  pair.first.throwable);

            throw ClusterHealServiceException.FAILED_CREATE_REPLACEMENT_VM(node
                  .getVmName());
         }
      } catch (InterruptedException e) {
         logger.error("error in fixing vm " + nodeName, e);
         throw BddException.INTERNAL(e, e.getMessage());
      }

      return null;
   }

   @Override
   public void updateData(String clusterName, String groupName,
         String nodeName, String newVmId) {
      NodeEntity node = clusterEntityMgr.findNodeByName(nodeName);

      logger.info("start update vm id and host info for node " + nodeName);
      VcVirtualMachine vm = ClusterUtil.getVcVm(clusterEntityMgr, node);

      node.setMoId(vm.getId());
      node.setHostName(vm.getHost().getName());
      clusterEntityMgr.update(node);

      logger.info("sync up status for node " + nodeName);
      clusterEntityMgr.syncUpNode(clusterName, nodeName);

      List<DiskEntity> fullDiskSet = clusterEntityMgr.getDisks(nodeName);
      for (DiskEntity disk : fullDiskSet) {
         VcVmUtil.populateDiskInfo(disk, newVmId);
      }
      clusterEntityMgr.updateDisks(nodeName, fullDiskSet);
   }

   @Override
   public boolean fixDiskFailures(String clusterName, String groupName,
         String nodeName, List<DiskSpec> fullDiskSet) {
      return true;
   }

   @Override
   public void verifyNodeStatus(String vmId, String nodeName) {
      NodeEntity nodeEntity = clusterEntityMgr.findNodeByName(nodeName);
      JobUtils.verifyNodeStatus(nodeEntity, NodeStatus.VM_READY, false, clusterEntityMgr);
   }

   @Override
   public void startVm(String nodeName, String vmId, String clusterName) {
      NodeEntity nodeEntity = clusterEntityMgr.findNodeByName(nodeName);
      StartVmPostPowerOn query =
            new StartVmPostPowerOn(nodeEntity.fetchAllPortGroups(),
                  Configuration.getInt(Constants.VM_POWER_ON_WAITING_SEC_KEY, Constants.VM_POWER_ON_WAITING_SEC),
                  clusterEntityMgr);

      VcVirtualMachine vcVm = ClusterUtil.getVcVm(clusterEntityMgr, nodeEntity);

      if (vcVm == null) {
         logger.error("VC vm does not exist for vmId: " + vmId);
      }

      VcHost host = vcVm.getHost();

      Callable<Void>[] storeProceduresArray = new Callable[1];
      storeProceduresArray[0] = new StartVmSP(vcVm, null, query, host);
      NoProgressUpdateCallback callback = new NoProgressUpdateCallback();

      try {
         ExecutionResult[] result =
               Scheduler
               .executeStoredProcedures(
                     com.vmware.aurora.composition.concurrent.Priority.BACKGROUND,
                     storeProceduresArray, callback);
         if (result == null) {
            logger.error("No result from composition layer");
         } else {
            if (result[0].finished && result[0].throwable == null) {
               logger.info("successfully power on vm for node: " + nodeName);
            } else {
               logger.error("failed in powering on vm for node: " + nodeName,
                     result[0].throwable);
               throw ClusterHealServiceException.FAILED_POWER_ON_VM(nodeName);
            }
         }
      } catch (InterruptedException e) {
         logger.error("error in run operation on vm.", e);
         throw ClusterHealServiceException.INTERNAL(e,
               "error in executing startVmSp");
      }
   }

   @Override
   @SuppressWarnings("unchecked")
   public VcVirtualMachine replaceBadDisksExceptSystem(String clusterName, String groupName, String nodeName, List<DiskSpec> replacementDisks) {

      ClusterCreate spec = configMgr.getClusterConfig(clusterName);
      NodeEntity node = clusterEntityMgr.findByName(spec.getName(), groupName, nodeName);

      List<DiskSpec> fullDiskList = getReplacedFullDisks(node.getVmName(), replacementDisks);

      VmSchema createSchema = VcVmUtil.getVmSchema(spec, groupName, fullDiskList, node.getNodeGroup().getCluster().getTemplateId(), Constants.ROOT_SNAPSTHOT_NAME);
      boolean isMapDistro = clusterEntityMgr.findByName(clusterName).getDistroVendor().equalsIgnoreCase(Constants.MAPR_VENDOR);

      ReplaceVmBadDisksSP replaceVmDisksPrePowerOnSP = new ReplaceVmBadDisksSP(node,
            createSchema.diskSchema, VcVmUtil.getTargetRp(spec.getName(), groupName, node),
            getTargetDatastore(fullDiskList),
            getBadDataDiskEntities(node.getVmName()), isMapDistro);

      try {
         Callable<Void>[] storeProceduresArray = new Callable[1];
         storeProceduresArray[0] = replaceVmDisksPrePowerOnSP;

         ExecutionResult[] result = Scheduler.executeStoredProcedures(com.vmware.aurora.composition.concurrent.Priority.BACKGROUND, storeProceduresArray, null);

         if (result == null) {
            logger.error("Failed to replace bad data disks for node " + nodeName);
            return null;
         }

         Throwable replacedDataDisksVmSpException = result[0].throwable;
         if (result[0].finished && replacedDataDisksVmSpException == null) {

            ReplaceVmBadDisksSP sp = (ReplaceVmBadDisksSP) storeProceduresArray[0];
            VcVirtualMachine vm = sp.getVm();
            AuAssert.check(vm != null);

            return vm;
         } else {
            logger.error("Failed to replace bad data disks for node " + node.getVmName(), replacedDataDisksVmSpException);
            throw ClusterHealServiceException.FAILED_TO_REPLACE_BAD_DATA_DISKS(node.getVmName());
         }

      } catch (InterruptedException e) {
         logger.error("error in fixing vm " + nodeName, e);
         throw BddException.INTERNAL(e, e.getMessage());
      }
   }

   @Override
   public VcVirtualMachine getFixingVm(String clusterName,
         String groupName, String nodeName) {
      NodeEntity node = clusterEntityMgr.findNodeByName(nodeName);

      VcResourcePool rp = VcVmUtil.getTargetRp(clusterName, groupName, node);

      String recoverVmName = node.getVmName() + RECOVERY_VM_NAME_POSTFIX;
      if (node.getMoId() != null) {
         VcVirtualMachine vm = getVmFromVc(node);

         // Destroy recover VM from VC
         destroyVMFromVc(rp, nodeName, recoverVmName);

         if (hasBadDisksExceptSystem(nodeName)) {
            return vm;
         } else {
            return null;
         }

      } else {
         return getFixingVmFromRp(rp, nodeName, recoverVmName);
      }
   }

   private void destroyVMFromVc(VcResourcePool rp, String nodeName, String recoverVmName) {
      VcVirtualMachine recoverVm = VcVmUtil.findVmInRp(rp, recoverVmName);
      if (recoverVm != null) {
         try {
            VcVmUtil.destroyVm(recoverVm.getId(), false);
         } catch (Exception e) {
            logger.error("failed to remove obsolete recovery vm for node " + nodeName);
            throw ClusterHealServiceException.FAILED_DELETE_VM(recoverVmName);
         }
      }
   }

   private VcVirtualMachine getVmFromVc(NodeEntity node) {
      VcVirtualMachine vm = VcCache.getIgnoreMissing(node.getMoId());
      // the vm id is null if the vm is removed
      if (vm == null) {
         throw ClusterHealServiceException.ERROR_STATUS(node.getVmName(),
               "Serengeti and VC are inconsistent as vm " + node.getVmName() + " is recorded in Seregeti, but not found in VC.");
      }
      return vm;
   }

   private VcVirtualMachine getFixingVmFromRp(VcResourcePool rp, String nodeName, String recoverVmName) {
      VcVirtualMachine oldVm = VcVmUtil.findVmInRp(rp, nodeName);
      VcVirtualMachine recoverVm = VcVmUtil.findVmInRp(rp, recoverVmName);

      if (oldVm != null && recoverVm != null) {
         destroyVMFromVc(rp, nodeName, recoverVmName);
         logger.info("Delete recover vm if " + "vm " + oldVm.getId() + " and recover vm " + recoverVm.getId() + " both exist");
         return null;
      } else if (oldVm == null && recoverVm == null) {
         logger.info("Original vm and recover vm both not exist.");
         return null;
      } else if (recoverVm != null) {
         logger.info("recover vm " + recoverVm.getId() + " exists, rename it to " + nodeName);
         VcVmUtil.rename(recoverVm.getId(), nodeName);
         return recoverVm;
      } else {
         logger.info("recovery probably failed at power on last time, simply return vm " + oldVm.getId());
         return oldVm;
      }
   }

   // replace bad disks with fixing disk, combining as a new disk set
   private List<DiskSpec> getReplacedFullDisks(String nodeName, List<DiskSpec> replacementDisks) {
      List<DiskSpec> fullDiskList = VcVmUtil.toDiskSpecList(clusterEntityMgr.getDisks(nodeName));

      for(DiskSpec diskSpec : fullDiskList) {
         //find the disk with same id, replace the target Ds
         for(DiskSpec replaceDiskEntity : replacementDisks) {
            if(diskSpec.getId() == replaceDiskEntity.getId()) {
               diskSpec.setTargetDs(replaceDiskEntity.getTargetDs());
               diskSpec.setVmdkPath(replaceDiskEntity.getVmdkPath());
               break;
            }
         }
      }

      return fullDiskList;
   }

   private Map<String, String> generateMachineId(ClusterCreate clusterSpec, NodeEntity node) {

      List<NetworkAdd> networkAdds = clusterSpec.getNetworkings();

      // this getRepalcementVmSP() is not called by any UT, so no need to check NPE for node.getIpConfigsInfo()
      GuestMachineIdSpec machineIdSpec = new GuestMachineIdSpec(networkAdds,
            node.fetchPortGroupToIpMap(), node.getPrimaryMgtNic().getNetworkEntity().getPortGroup(), node, networkMgr);
      logger.info("machine id of vm " + node.getVmName() + ":\n" + machineIdSpec.toString());
      Map<String, String> guestVariable = machineIdSpec.toGuestVariable();

      return guestVariable;
   }

   private boolean getHaFlag(ClusterCreate clusterSpec, String groupName) {

      boolean ha = false;
      String haFlag = clusterSpec.getNodeGroup(groupName).getHaFlag();

      if (haFlag != null && Constants.HA_FLAG_ON.equals(haFlag.toLowerCase())) {
         ha = true;
      }

      if (haFlag != null && Constants.HA_FLAG_FT.equals(haFlag.toLowerCase())) {
         ha = true;
      }

      return ha;
   }

   private boolean getFtFlag(ClusterCreate clusterSpec, String groupName) {

      boolean ft = false;

      String haFlag = clusterSpec.getNodeGroup(groupName).getHaFlag();
      if (haFlag != null && Constants.HA_FLAG_FT.equals(haFlag.toLowerCase())) {
         ft = true;
      }

      return ft;
   }
}