DeltaPropagationStatsDUnitTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
/**
 * 
 */
package org.apache.geode.internal.cache;

import static org.apache.geode.distributed.ConfigurationProperties.*;
import static org.junit.Assert.*;

import java.util.Properties;

import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;

import org.apache.geode.DeltaTestImpl;
import org.apache.geode.InvalidDeltaException;
import org.apache.geode.cache.AttributesFactory;
import org.apache.geode.cache.Cache;
import org.apache.geode.cache.CacheFactory;
import org.apache.geode.cache.DataPolicy;
import org.apache.geode.cache.EntryEvent;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.RegionAttributes;
import org.apache.geode.cache.Scope;
import org.apache.geode.cache.client.Pool;
import org.apache.geode.cache.client.PoolManager;
import org.apache.geode.cache.server.CacheServer;
import org.apache.geode.cache.util.CacheListenerAdapter;
import org.apache.geode.distributed.DistributedSystem;
import org.apache.geode.distributed.internal.DistributionStats;
import org.apache.geode.internal.AvailablePort;
import org.apache.geode.internal.cache.tier.sockets.CacheClientNotifier;
import org.apache.geode.internal.cache.tier.sockets.CacheClientProxy;
import org.apache.geode.internal.tcp.ConnectionTable;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.NetworkUtils;
import org.apache.geode.test.dunit.VM;
import org.apache.geode.test.dunit.Wait;
import org.apache.geode.test.dunit.WaitCriterion;
import org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase;
import org.apache.geode.test.junit.categories.DistributedTest;

@Category(DistributedTest.class)
public class DeltaPropagationStatsDUnitTest extends JUnit4DistributedTestCase {

  protected static VM vm0 = null;

  protected static VM vm1 = null;

  protected static VM vm2 = null;

  protected static VM vm3 = null;

  protected static Cache cache = null;

  protected static Pool pool = null;

  public static String REGION_NAME = "DeltaPropagationStatsDUnitTest";

  private static String DELTA_KEY = "DELTA_KEY_";

  private static String LAST_KEY = "LAST_KEY";

  private static boolean lastKeyReceived = false;

  private static final int PEER_TO_PEER = 1;

  private static final int SERVER_TO_CLIENT = 2;

  private static final int CLIENT_TO_SERVER = 3;

  @Override
  public final void postSetUp() throws Exception {
    final Host host = Host.getHost(0);
    vm0 = host.getVM(0);
    vm1 = host.getVM(1);
    vm2 = host.getVM(2);
    vm3 = host.getVM(3);
  }

  @Override
  public final void preTearDown() throws Exception {
    lastKeyReceived = false;
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.resetLastKeyReceived());
    vm1.invoke(() -> DeltaPropagationStatsDUnitTest.resetLastKeyReceived());
    vm2.invoke(() -> DeltaPropagationStatsDUnitTest.resetLastKeyReceived());
    vm3.invoke(() -> DeltaPropagationStatsDUnitTest.resetLastKeyReceived());
    closeCache();
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.closeCache());
    vm1.invoke(() -> DeltaPropagationStatsDUnitTest.closeCache());
    vm2.invoke(() -> DeltaPropagationStatsDUnitTest.closeCache());
    vm3.invoke(() -> DeltaPropagationStatsDUnitTest.closeCache());
  }

  public static void resetLastKeyReceived() {
    lastKeyReceived = false;
  }

  public static void closeCache() {
    if (cache != null && !cache.isClosed()) {
      cache.close();
      cache.getDistributedSystem().disconnect();
    }
  }

  /**
   * No error or resending of delta.
   * 
   * @throws Exception
   */
  @Test
  public void testS2CDeltaPropagationCleanStats() throws Exception {
    int numOfKeys = 50;
    long updates = 50;
    Object args[] =
        new Object[] {Boolean.TRUE, DataPolicy.REPLICATE, Scope.DISTRIBUTED_ACK, Boolean.TRUE};
    int port =
        (Integer) vm0.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);

    createClientCache(NetworkUtils.getServerHostName(vm0.getHost()), port);

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putCleanDelta(Integer.valueOf(numOfKeys),
        Long.valueOf(updates)));
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putLastKey());

    waitForLastKey();

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaSenderStats(
        Integer.valueOf(SERVER_TO_CLIENT), Long.valueOf(numOfKeys * updates)));
    verifyDeltaReceiverStats(SERVER_TO_CLIENT, numOfKeys * updates, 0L);
  }

  /**
   * Simulates error in fromDelta() and toDelta()
   * 
   * @throws Exception
   */
  @Test
  public void testS2CDeltaPropagationFailedStats1() throws Exception {
    int numOfKeys = 25;
    long updates = 50;
    long errors = 100, errors2 = 34;
    Object args[] =
        new Object[] {Boolean.TRUE, DataPolicy.REPLICATE, Scope.DISTRIBUTED_ACK, Boolean.TRUE};
    int port =
        (Integer) vm0.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);

    createClientCache(NetworkUtils.getServerHostName(vm0.getHost()), port);

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putErrorDeltaForReceiver(
        Integer.valueOf(numOfKeys), Long.valueOf(updates), Long.valueOf(errors)));
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putErrorDeltaForSender(
        Integer.valueOf(numOfKeys), Long.valueOf(updates), Long.valueOf(errors2), Boolean.FALSE));
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putLastKey());

    waitForLastKey();

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaSenderStats(
        Integer.valueOf(SERVER_TO_CLIENT), Long.valueOf(2 * numOfKeys * updates - errors2)));
    verifyDeltaReceiverStats(SERVER_TO_CLIENT, 2 * numOfKeys * updates - errors - errors2, errors);
  }

  /**
   * Simulates old value null, entry null, InvalidDeltaExeption
   * 
   * @throws Exception
   */
  @Ignore("TODO")
  @Test
  public void testS2CDeltaPropagationFailedStats2() throws Exception {}

  /**
   * No error or resending of delta.
   * 
   * @throws Exception
   */
  @Test
  public void testP2PDeltaPropagationCleanStats() throws Exception {
    int numOfKeys = 50;
    long updates = 50;
    Object args[] =
        new Object[] {Boolean.TRUE, DataPolicy.REPLICATE, Scope.DISTRIBUTED_ACK, Boolean.TRUE};
    vm0.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    vm1.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    vm2.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    // Only delta should get sent to vm1 and vm2
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putCleanDelta(Integer.valueOf(numOfKeys),
        Long.valueOf(updates)));
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putLastKey());

    vm1.invoke(() -> DeltaPropagationStatsDUnitTest.waitForLastKey());
    vm2.invoke(() -> DeltaPropagationStatsDUnitTest.waitForLastKey());

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest
        .verifyDeltaSenderStats(Integer.valueOf(PEER_TO_PEER), Long.valueOf(numOfKeys * updates)));
    vm1.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaReceiverStats(
        Integer.valueOf(PEER_TO_PEER), Long.valueOf(numOfKeys * updates), Long.valueOf(0)));
    vm2.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaReceiverStats(
        Integer.valueOf(PEER_TO_PEER), Long.valueOf(numOfKeys * updates), Long.valueOf(0)));
  }

  /**
   * Simulates error in fromDelta()
   * 
   * @throws Exception
   */
  @Test
  public void testP2PDeltaPropagationFailedStats1() throws Exception {
    int numOfKeys = 50, numOfkeys2 = 10;
    long updates = 50, updates2 = 50;
    long errors = 100, errors2 = 0;
    Object args[] =
        new Object[] {Boolean.TRUE, DataPolicy.REPLICATE, Scope.DISTRIBUTED_ACK, Boolean.TRUE};
    vm0.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    vm1.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    vm2.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    // Only delta should get sent to vm1 and vm2
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putErrorDeltaForReceiver(
        Integer.valueOf(numOfKeys), Long.valueOf(updates), Long.valueOf(errors)));
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putErrorDeltaForSender(
        Integer.valueOf(numOfkeys2), Long.valueOf(updates2), Long.valueOf(errors2), Boolean.FALSE));
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.putLastKey());

    vm1.invoke(() -> DeltaPropagationStatsDUnitTest.waitForLastKey());
    vm2.invoke(() -> DeltaPropagationStatsDUnitTest.waitForLastKey());

    long deltasSent = (numOfKeys * updates) + (numOfkeys2 * updates2) - errors2;
    long deltasProcessed = deltasSent - errors;

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest
        .verifyDeltaSenderStats(Integer.valueOf(PEER_TO_PEER), Long.valueOf(deltasSent)));
    vm1.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaReceiverStats(
        Integer.valueOf(PEER_TO_PEER), Long.valueOf(deltasProcessed), Long.valueOf(errors)));
    vm2.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaReceiverStats(
        Integer.valueOf(PEER_TO_PEER), Long.valueOf(deltasProcessed), Long.valueOf(errors)));
  }

  /**
   * Simulates old value null, entry null
   * 
   * @throws Exception
   */
  @Ignore("TODO")
  @Test
  public void testP2PDeltaPropagationFailedStats2() throws Exception {}

  /**
   * No error or resending of delta.
   * 
   * @throws Exception
   */
  @Test
  public void testC2SDeltaPropagationCleanStats() throws Exception {
    int numOfKeys = 50;
    long updates = 50;

    Object args[] =
        new Object[] {Boolean.TRUE, DataPolicy.REPLICATE, Scope.DISTRIBUTED_ACK, Boolean.TRUE};
    Integer port =
        (Integer) vm0.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    createClientCache(NetworkUtils.getServerHostName(vm0.getHost()), port);

    putCleanDelta(numOfKeys, updates);
    putLastKey();

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.waitForLastKey());

    verifyDeltaSenderStats(CLIENT_TO_SERVER, numOfKeys * updates);
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaReceiverStats(
        Integer.valueOf(CLIENT_TO_SERVER), Long.valueOf(numOfKeys * updates), Long.valueOf(0)));

    // Unrelated to Delta feature. Piggy-backing on existing test code
    // to validate fix for #49539.
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.doPuts(numOfKeys));
    long clientOriginatedEvents = numOfKeys * updates + numOfKeys + 1;
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.verifyCCPStatsBug49539(clientOriginatedEvents));
  }

  /**
   * Simulates error in fromDelta() and toDelta()
   * 
   * @throws Exception
   */
  @Test
  public void testC2SDeltaPropagationFailedStats1() throws Exception {
    int numOfKeys = 50;
    long updates = 50;
    long errors = 100, errors2 = 13;

    Object args[] =
        new Object[] {Boolean.TRUE, DataPolicy.REPLICATE, Scope.DISTRIBUTED_ACK, Boolean.TRUE};
    Integer port =
        (Integer) vm0.invoke(DeltaPropagationStatsDUnitTest.class, "createServerCache", args);
    createClientCache(NetworkUtils.getServerHostName(vm0.getHost()), port);

    putErrorDeltaForReceiver(numOfKeys, updates, errors);
    putErrorDeltaForSender(numOfKeys, updates, errors2, Boolean.FALSE);
    putLastKey();

    long deltasSent = 2 * (numOfKeys * updates) - errors2;
    long deltasProcessed = deltasSent - errors;

    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.waitForLastKey());

    verifyDeltaSenderStats(CLIENT_TO_SERVER, deltasSent);
    vm0.invoke(() -> DeltaPropagationStatsDUnitTest.verifyDeltaReceiverStats(
        Integer.valueOf(CLIENT_TO_SERVER), Long.valueOf(deltasProcessed), Long.valueOf(errors)));
  }

  /**
   * Simulates old value null, entry null
   * 
   * @throws Exception
   */
  @Ignore("TODO")
  @Test
  public void testC2SDeltaPropagationFailedStats2() throws Exception {}

  public static void waitForLastKey() {
    WaitCriterion wc = new WaitCriterion() {
      public boolean done() {
        return lastKeyReceived;
      }

      public String description() {
        return "Last key NOT received.";
      }
    };
    Wait.waitForCriterion(wc, 15 * 1000, 100, true);
  }

  public static void putCleanDelta(Integer keys, Long updates) {
    Region r = cache.getRegion(REGION_NAME);

    for (int i = 0; i < keys; i++) {
      r.create(DELTA_KEY + i, new DeltaTestImpl());
    }

    for (int i = 0; i < keys; i++) {
      for (long j = 0; j < updates; j++) {
        DeltaTestImpl delta = new DeltaTestImpl();
        if (j % 3 == 1) {
          delta.setIntVar(10);
        } else if (j % 3 == 2) {
          delta.setStr("two");
        } else {
          delta.setByteArr(new byte[] {11, 22, 33, 44});
          delta.setDoubleVar(5.7);
        }
        r.put(DELTA_KEY + i, delta);
      }
    }
  }

  public static void putErrorDeltaForReceiver(Integer keys, Long updates, Long errors) {
    Region r = cache.getRegion(REGION_NAME);
    assertTrue("Errors cannot be more than 1/3rd of total udpates", (updates * keys) / 3 > errors);

    for (int i = 0; i < keys; i++) {
      r.create(DELTA_KEY + i, new DeltaTestImpl());
    }

    for (int i = 0; i < keys; i++) {
      for (long j = 0; j < updates; j++) {
        DeltaTestImpl delta = new DeltaTestImpl();
        if (j % 3 == 1) {
          delta.setIntVar(10);
        } else if (j % 3 == 2) {
          delta.setStr("two");
          if (errors != 0) {
            delta.setStr(DeltaTestImpl.ERRONEOUS_STRING_FOR_FROM_DELTA);
            errors--;
          }
        } else {
          delta.setByteArr(new byte[] {11, 22, 33, 44});
          delta.setDoubleVar(5.7);
        }
        r.put(DELTA_KEY + i, delta);
      }
    }
    assertTrue("Error puts not exhausted", errors == 0);
  }

  public static void putErrorDeltaForSender(Integer keys, Long updates, Long errors,
      Boolean doCreate) {
    Region r = cache.getRegion(REGION_NAME);
    assertTrue("Errors cannot be more than 1/3rd of total updates", (keys * updates) / 3 > errors);

    if (doCreate) {
      for (int i = 0; i < keys; i++) {
        r.create(DELTA_KEY + i, new DeltaTestImpl());
      }
    }

    for (int i = 0; i < keys; i++) {
      for (long j = 0; j < updates; j++) {
        DeltaTestImpl delta = new DeltaTestImpl();
        if (j % 3 == 1) {
          delta.setStr("one");
        } else if (j % 3 == 2) {
          delta.setIntVar(111);
          if (errors != 0) {
            delta.setIntVar(DeltaTestImpl.ERRONEOUS_INT_FOR_TO_DELTA);
            errors--;
          }
        } else {
          delta.setByteArr(new byte[] {11, 22, 33, 44});
          delta.setDoubleVar(5.7);
        }
        try {
          r.put(DELTA_KEY + i, delta);
          assertTrue("Expected an InvalidDeltaException to be thrown, but it wasn't!",
              delta.getIntVar() != DeltaTestImpl.ERRONEOUS_INT_FOR_TO_DELTA);
        } catch (InvalidDeltaException ide) {
          assertTrue("InvalidDeltaException not expected.",
              delta.getIntVar() == DeltaTestImpl.ERRONEOUS_INT_FOR_TO_DELTA);
          cache.getLoggerI18n().fine("Recieved InvalidDeltaException as expected.");
        }
      }
    }
    assertTrue("Error puts not exhausted", errors == 0);
  }

  public static void doPuts(Integer num) {
    Region r = cache.getRegion(REGION_NAME);
    for (int i = 0; i < num; i++) {
      r.put("SAMPLE_" + i, "SAMPLE_" + i);
    }
  }

  public static void putLastKey() {
    Region r = cache.getRegion(REGION_NAME);
    r.create(LAST_KEY, "LAST_VALUE");
  }

  public static void verifyDeltaSenderStats(Integer path, Long expectedDeltasSent) {
    long numOfDeltasSent = 0;
    long deltaTime = 0;
    LocalRegion region = (LocalRegion) cache.getRegion(REGION_NAME);
    if (path == PEER_TO_PEER) {
      numOfDeltasSent = region.getCachePerfStats().getDeltasSent();
      deltaTime = region.getCachePerfStats().getDeltasPreparedTime();
    } else if (path == SERVER_TO_CLIENT) {
      CacheClientNotifier ccn = ((CacheServerImpl) cache.getCacheServers().toArray()[0])
          .getAcceptor().getCacheClientNotifier();

      numOfDeltasSent = ((CacheClientProxy) ccn.getClientProxies().toArray()[0]).getStatistics()
          .getDeltaMessagesSent();
      deltaTime = 1; // dummy assignment
    } else if (path == CLIENT_TO_SERVER) {
      numOfDeltasSent = region.getCachePerfStats().getDeltasSent();
      if (DistributionStats.enableClockStats) {
        deltaTime = region.getCachePerfStats().getDeltasPreparedTime();
      } else {
        deltaTime = 1; // dummy assignment
      }
    } else {
      fail("Invalid path code for delta propagation: " + path);
    }
    assertTrue(
        "Number of deltas sent was expected to be " + expectedDeltasSent + " but is "
            + numOfDeltasSent,
        numOfDeltasSent == expectedDeltasSent
            // C2S are intermittently failing with 1 less delta sent so allowing it
            || numOfDeltasSent + 1 == expectedDeltasSent);

    // see bug #41879
    // assertTrue("Delta calculation is expected to take non-zero time",
    // deltaTime > 0);
  }

  public static void verifyDeltaReceiverStats(Integer path, Long expectedDeltasProcessed,
      Long expectedDeltaFailures) {
    long numOfDeltasProcessed = 0;
    long deltaTime = 0;
    long deltaFailures = 0;
    if (path == PEER_TO_PEER || path == CLIENT_TO_SERVER || path == SERVER_TO_CLIENT) {
      CachePerfStats stats = ((DistributedRegion) cache.getRegion(REGION_NAME)).getCachePerfStats();

      numOfDeltasProcessed = stats.getDeltaUpdates();
      deltaTime = stats.getDeltaUpdatesTime();
      deltaFailures = stats.getDeltaFailedUpdates();
    } else {
      fail("Invalid path code for delta propagation: " + path);
    }
    assertTrue("Number of deltas received was expected to be " + expectedDeltasProcessed
        + " but is " + numOfDeltasProcessed, numOfDeltasProcessed == expectedDeltasProcessed);

    // It is possible for deltaTime to be zero depending on the system clock
    // resolution
    assertTrue("Delta calculation is expected to be >= zero but was " + deltaTime, deltaTime >= 0);

    assertTrue("Number of delta failures was expected to be " + expectedDeltaFailures + " but is "
        + deltaFailures, deltaFailures == expectedDeltaFailures);
  }

  public static void verifyCCPStatsBug49539(Long expected) {
    int actual = CacheClientNotifier.getInstance().getClientProxies().iterator().next()
        .getStatistics().getMessagesNotQueuedOriginator();
    assertEquals("Bug #49539: stats do not match, expected messageNotQueuedOriginator: " + expected
        + ", actual: " + actual, (long) expected, actual);
  }

  public static void createClientCache(String host, Integer port) throws Exception {
    Properties props = new Properties();
    props.setProperty(MCAST_PORT, "0");
    props.setProperty(LOCATORS, "");
    cache = new DeltaPropagationStatsDUnitTest().createCache(props);
    pool = PoolManager.createFactory().addServer(host, port).setThreadLocalConnections(true)
        .setMinConnections(1).setSubscriptionEnabled(true).setSubscriptionRedundancy(0)
        .setReadTimeout(10000).setSocketBufferSize(32768)
        .create("DeltaPropagationStatsDunitTestPool");
    AttributesFactory factory = new AttributesFactory();
    factory.setScope(Scope.DISTRIBUTED_ACK);
    factory.setDataPolicy(DataPolicy.NORMAL);
    factory.setPoolName(pool.getName());
    factory.setCloningEnabled(false);

    factory.addCacheListener(new CacheListenerAdapter() {
      public void afterCreate(EntryEvent event) {
        if (LAST_KEY.equals(event.getKey())) {
          lastKeyReceived = true;
        }
      }
    });

    RegionAttributes attrs = factory.create();
    cache.createRegion(REGION_NAME, attrs).registerInterest("ALL_KEYS");
  }

  public static Integer createServerCache(Boolean flag) throws Exception {
    ConnectionTable.threadWantsSharedResources();
    return createServerCache(flag, DataPolicy.DEFAULT, Scope.DISTRIBUTED_ACK, false);
  }

  public static Integer createServerCache(Boolean flag, DataPolicy policy, Scope scope,
      Boolean listener) throws Exception {
    ConnectionTable.threadWantsSharedResources();
    DeltaPropagationStatsDUnitTest test = new DeltaPropagationStatsDUnitTest();
    Properties props = new Properties();
    if (!flag) {
      props.setProperty(DELTA_PROPAGATION, "false");
    }
    cache = test.createCache(props);
    AttributesFactory factory = new AttributesFactory();
    factory.setScope(scope);
    factory.setDataPolicy(policy);

    if (listener) {
      factory.addCacheListener(new CacheListenerAdapter() {
        public void afterCreate(EntryEvent event) {
          if (event.getKey().equals(LAST_KEY)) {
            lastKeyReceived = true;
          }
        }
      });
    }

    Region region = cache.createRegion(REGION_NAME, factory.create());
    if (!policy.isReplicate()) {
      region.create("KEY", "KEY");
    }
    region.getAttributesMutator().setCloningEnabled(false);
    CacheServer server = cache.addCacheServer();
    int port = AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
    server.setPort(port);
    server.setNotifyBySubscription(true);
    server.start();
    return server.getPort();
  }

  private Cache createCache(Properties props) throws Exception {
    DistributedSystem ds = getSystem(props);
    ds.disconnect();
    ds = getSystem(props);
    Cache result = null;
    result = CacheFactory.create(ds);
    if (result == null) {
      throw new Exception("CacheFactory.create() returned null");
    }
    return result;
  }

}