/* * Copyright (c) 2015 EMC Corporation * All Rights Reserved */ package com.emc.storageos.db.server.impl; import java.util.List; import org.apache.cassandra.service.StorageService; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.emc.storageos.coordinator.client.model.Constants; import com.emc.storageos.coordinator.client.model.Site; import com.emc.storageos.coordinator.client.model.SiteInfo; import com.emc.storageos.coordinator.client.model.SiteMonitorResult; import com.emc.storageos.coordinator.client.model.SiteState; import com.emc.storageos.coordinator.client.service.CoordinatorClient; import com.emc.storageos.coordinator.client.service.DrUtil; import com.emc.storageos.coordinator.common.Configuration; import com.emc.storageos.coordinator.common.Service; import com.emc.storageos.db.common.DbConfigConstants; import com.emc.storageos.security.audit.AuditLogManager; import com.emc.storageos.services.OperationTypeEnum; import com.emc.storageos.svcs.errorhandling.resources.APIException; /** * Thread to rebuild the local db node if the local site is in STANDBY_SYNCING state. * Update the local site state to STANDBY_SYNCED when db rebuild finishes for both db and geodb on all nodes. */ public class DbRebuildRunnable implements Runnable { private static final Logger log = LoggerFactory.getLogger(DbRebuildRunnable.class); private CoordinatorClient coordinator; private int nodeCount; private Service service; private volatile boolean isRunning; public void setCoordinator(CoordinatorClient coordinator) { this.coordinator = coordinator; } public void setNodeCount(int nodeCount) { this.nodeCount = nodeCount; } public void setService(Service service) { this.service = service; } @Override public void run() { if (isRunning) { log.info("db rebuild in progress, nothing to do"); return; } DrUtil drUtil = new DrUtil(coordinator); Site localSite = drUtil.getLocalSite(); if (!localSite.getState().equals(SiteState.STANDBY_SYNCING)) { log.info("db in sync, nothing to do"); return; } Configuration dbconfig = coordinator.queryConfiguration(coordinator.getSiteId(), coordinator.getVersionedDbConfigPath(service.getName(), service.getVersion()), service.getId()); if (isLastDataSyncCurrent(dbconfig)) { log.info("last data sync time is later than the target site info update, nothing to do"); return; } Site primarySite = drUtil.getActiveSite(); String sourceDc = drUtil.getCassandraDcId(primarySite); log.info("starting db rebuild from source dc {}", sourceDc); isRunning = true; StorageService.instance.rebuild(sourceDc); long currentSyncTime = System.currentTimeMillis(); log.info("local db rebuild finishes. Updating last data sync time to {}", currentSyncTime); dbconfig.setConfig(DbConfigConstants.LAST_DATA_SYNC_TIME, String.valueOf(currentSyncTime)); coordinator.persistServiceConfiguration(coordinator.getSiteId(), dbconfig); if (dbRebuildComplete(Constants.DBSVC_NAME) && dbRebuildComplete(Constants.GEODBSVC_NAME)) { localSite = drUtil.getLocalSite(); // update the site state only if it's still STANDBY_SYNCING // do nothing if it gets set to STANDBY_ERROR earlier if (localSite.getState().equals(SiteState.STANDBY_SYNCING)) { purgeOldDataRevision(drUtil); // reset heartbeat for this site SiteMonitorResult dbHeartbeat = coordinator.getTargetInfo(localSite.getUuid(), SiteMonitorResult.class); if (dbHeartbeat != null) { dbHeartbeat.setDbQuorumLostSince(0); coordinator.setTargetInfo(localSite.getUuid(), dbHeartbeat); log.info("Reset db heartbeat state for {}", localSite.getUuid()); } log.info("all db rebuild finish, updating site state to STANDBY_SYNCED"); localSite.setState(SiteState.STANDBY_SYNCED); coordinator.persistServiceConfiguration(localSite.toConfiguration()); } } isRunning = false; } private void purgeOldDataRevision(DrUtil drUtil) { try { coordinator.startTransaction(); log.info("Purge old data revisions after resync data from active site"); long vdcTargetVersion = DrUtil.newVdcConfigVersion(); String localSiteId = drUtil.getLocalSite().getUuid(); for (Site site : drUtil.listSites()) { drUtil.updateVdcTargetVersion(site.getUuid(), SiteInfo.DR_OP_PURGE_DATA_REVISION, vdcTargetVersion, localSiteId, localSiteId); } coordinator.commitTransaction(); } catch (Exception e) { log.warn("Failed to purge old data revision on local" , e); coordinator.discardTransaction(); } } private boolean dbRebuildComplete(String svcName) { List<Configuration> configs = coordinator.queryAllConfiguration(coordinator.getSiteId(), coordinator.getVersionedDbConfigPath(svcName, service.getVersion())); int count = 0; for (Configuration config : configs) { if (isLastDataSyncCurrent(config)) { count++; } } return (count == nodeCount); } // check if the last data sync time is later than the target site info update private boolean isLastDataSyncCurrent(Configuration dbConfig) { SiteInfo targetSiteInfo = coordinator.getTargetInfo(SiteInfo.class); String value = dbConfig.getConfig(DbConfigConstants.LAST_DATA_SYNC_TIME); return value != null && Long.valueOf(value) > targetSiteInfo.getVdcConfigVersion(); } }