ClusterValidationStage.java

package org.radargun.stages.cache;

import java.util.List;

import org.radargun.DistStageAck;
import org.radargun.StageResult;
import org.radargun.config.Property;
import org.radargun.config.Stage;
import org.radargun.stages.AbstractDistStage;
import org.radargun.state.SlaveState;
import org.radargun.traits.BasicOperations;
import org.radargun.traits.InjectTrait;
import org.radargun.utils.Utils;

/**
 * Distributed stage that would validate that cluster is correctly formed.
 * <pre>
 * Algorithm:
 * - each slave does a put(slaveIndex);
 * - each slave checks whether all (or part) of the remaining slaves replicated here.
 *
 * Config:
 *   - 'partialReplication' : is set to true, then the slave will consider that the cluster is formed when one slave
 *      replicated here. If false (default value) then replication will only be considered successful if all
 * (clusterSize)
 *      slaves replicated here.
 * </pre>
 *
 * @author Mircea Markus <Mircea.Markus@jboss.com>
 */
@Stage(doc = "Verifies that the cluster is formed by injecting an entry into the cache and then reading it from other nodes.")
public class ClusterValidationStage extends AbstractDistStage {

   private static final String KEY = "_InstallBenchmarkStage_";
   private static final String CONFIRMATION_KEY = "_confirmation_";

   @Property(doc = "If set to true, then the slave will consider that the cluster is formed when one slave " +
      "replicated the control entry. Otherwise the replication will only be considered successful if all " +
      "slaves replicated the control value. Default is false.")
   private boolean partialReplication = false;

   @Property(doc = "How many times we should try to retrieve the control entry.")
   private int replicationTryCount = 60;

   @Property(doc = "Delay between attempts to retrieve the control entry.")
   private int replicationTimeSleep = 2000;

   @InjectTrait
   private BasicOperations basicOperations;

   private BasicOperations.Cache cache;

   public DistStageAck executeOnSlave() {
      try {
         if (!isServiceRunning()) {
            log.info("Missing wrapper, not participating on validation");
            return successfulResponse();
         }
         cache = basicOperations.getCache(null);
         int replResult = checkReplicationSeveralTimes();
         if (!partialReplication) {
            if (replResult > 0) {//only executes this on the slaves on which replication happened.
               int index = confirmReplication();
               if (index >= 0) {
                  return errorResponse("Slave with index " + index + " hasn't confirmed the replication");
               }
            }
         } else {
            log.info("Using partial replication, skipping confirm phase");
         }
         return new ReplicationAck(slaveState, replResult);
      } catch (Exception e) {
         return errorResponse("Exception thrown", e);
      }
   }

   private int confirmReplication() throws Exception {
      cache.put(confirmationKey(slaveState.getSlaveIndex()), "true");
      for (int i : getExecutingSlaves()) {
         for (int j = 0; j < 10 && (cache.get(confirmationKey(i)) == null); j++) {
            tryToPut();
            cache.put(confirmationKey(slaveState.getSlaveIndex()), "true");
            Thread.sleep(1000);
         }
         if (cache.get(confirmationKey(i)) == null) {
            log.warn("Confirm phase unsuccessful. Slave " + i + " hasn't acknowledged the test");
            return i;
         }
      }
      log.info("Confirm phase successful.");
      return -1;
   }

   private String confirmationKey(int slaveIndex) {
      return CONFIRMATION_KEY + slaveIndex;
   }

   public StageResult processAckOnMaster(List<DistStageAck> acks) {
      logDurationInfo(acks);
      StageResult result = StageResult.SUCCESS;
      for (DistStageAck ack : acks) {
         if (ack.isError()) {
            log.warn("Ack error from remote slave: " + ack);
            result = errorResult();
         }
         if (!(ack instanceof ReplicationAck)) {
            log.info("Slave " + ack.getSlaveIndex() + " did not sent any response");
            continue;
         }
         ReplicationAck replicationAck = (ClusterValidationStage.ReplicationAck) ack;
         if (partialReplication) {
            if (replicationAck.result <= 0) {
               log.warn("Replication hasn't occurred on slave: " + ack);
               result = errorResult();
            }
         } else { //total replication expected
            int expectedRepl = getExecutingSlaves().size() - 1;
            if (!(replicationAck.result == expectedRepl)) {
               log.warn("On slave " + ack + " total replication hasn't occurred. Expected " + expectedRepl + " and received " + replicationAck.result);
               result = errorResult();
            }
         }
      }
      if (result.isError()) log.warn("Cluster hasn't formed!");
      return result;
   }


   private void tryToPut() throws Exception {
      int tryCount = 0;
      while (tryCount < 5) {
         try {
            cache.put(key(slaveState.getSlaveIndex()), "true");
            return;
         } catch (Throwable e) {
            log.warn("Error while trying to put data: ", e);
            tryCount++;
         }
      }
      throw new Exception("Couldn't accomplish addition before replication!");
   }


   private int checkReplicationSeveralTimes() throws Exception {
      tryToPut();
      int replCount = 0;
      for (int i = 0; i < replicationTryCount; i++) {
         replCount = replicationCount();
         if ((partialReplication && replCount >= 1) || (!partialReplication && (replCount == getExecutingSlaves().size() - 1))) {
            log.info("Replication test successfully passed. partialReplication? " + partialReplication + ", replicationCount = " + replCount);
            return replCount;
         }
         //adding our stuff one more time
         tryToPut();
         log.info("Replication test failed, " + (i + 1) + " tries so far. Sleeping for " + Utils.prettyPrintMillis(replicationTimeSleep)
            + " and trying again.");
         Thread.sleep(replicationTimeSleep);
      }
      log.info("Replication test failed. Last replication count is " + replCount);
      return -1;
   }

   private int replicationCount() throws Exception {
      int clusterSize = slaveState.getClusterSize();
      int replicaCount = 0;
      for (int i = 0; i < clusterSize; i++) {
         int currentSlaveIndex = slaveState.getSlaveIndex();
         if (i == currentSlaveIndex) {
            continue;
         }
         Object data = tryGet(i);
         if (data == null || !"true".equals(data.toString())) {
            log.trace("Cache with index " + i + " did *NOT* replicate");
         } else {
            log.trace("Cache with index " + i + " replicated here ");
            replicaCount++;
         }
      }
      log.info("Number of caches that replicated here is " + replicaCount);
      return replicaCount;
   }


   private Object tryGet(int i) throws Exception {
      int tryCont = 0;
      while (tryCont < 5) {
         try {
            return cache.get(key(i));
         } catch (Throwable e) {
            tryCont++;
         }
      }
      return null;
   }

   private String key(int slaveIndex) {
      return KEY + slaveIndex;
   }

   private static class ReplicationAck extends DistStageAck {
      final int result;

      private ReplicationAck(SlaveState slaveState, int result) {
         super(slaveState);
         this.result = result;
      }
   }
}