BloombergTickWriterTest.java

/**
 * Copyright (C) 2009 - present by OpenGamma Inc. and the OpenGamma group of companies
 *
 * Please see distribution for license.
 */
package com.opengamma.bbg.replay;

import static com.opengamma.bbg.replay.BloombergTick.FIELDS_KEY;
import static org.testng.AssertJUnit.assertTrue;

import java.io.File;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.SystemUtils;
import org.fudgemsg.FudgeContext;
import org.fudgemsg.FudgeMsg;
import org.fudgemsg.test.FudgeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import org.threeten.bp.Clock;
import org.threeten.bp.ZonedDateTime;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.opengamma.OpenGammaRuntimeException;
import com.opengamma.bbg.replay.BloombergTicksReplayer.Mode;
import com.opengamma.bbg.test.BloombergTestUtils;
import com.opengamma.util.fudgemsg.OpenGammaFudgeContext;
import com.opengamma.util.test.TestGroup;

/**
 * Test.
 */
@Test(groups = TestGroup.UNIT_SLOW)
public class BloombergTickWriterTest {

  private static final Logger s_logger = LoggerFactory.getLogger(BloombergTickWriterTest.class);

  private static final FudgeContext s_fudgeContext = OpenGammaFudgeContext.getInstance();

  private static final int TICKS_GENERATOR_THREAD_SIZE = 1;
  private static final int RUN_DURATION = 5000;
  private static final long REPORT_INTERVAL = RUN_DURATION * 3;
  private static final long WRITER_SPEED_THRESHOLD = 1024000; 
  private static final int MAX_QUEUE_SIZE = 1000;

  private BlockingQueue<FudgeMsg> _allTicksQueue = new ArrayBlockingQueue<FudgeMsg>(MAX_QUEUE_SIZE);
  private BloombergTickWriter _writer;
  private File _rootDir = new File(SystemUtils.getJavaIoTmpDir(), "tickDataTest");
  private RandomTicksGeneratorJob _ticksGenerator;
  private Map<String, String> _ticker2buid = ImmutableMap.of("QQQQ US Equity", "EQ0082335400001000");

  @BeforeMethod
  public void setUp(Method m) throws Exception {
    _writer = new BloombergTickWriter(s_fudgeContext, _allTicksQueue, _ticker2buid, _rootDir.getAbsolutePath(), StorageMode.MULTI);
    _ticksGenerator = new RandomTicksGeneratorJob(new ArrayList<String>(_ticker2buid.keySet()), _allTicksQueue);
    makeRootDir();
  }

  private void makeRootDir() {
    if (!_rootDir.exists()) {
      if(!_rootDir.mkdirs()) {
        throw new OpenGammaRuntimeException("unable to create root dir " + _rootDir);
      }
    } 
  }

  @AfterMethod
  public void tearDown() throws Exception {
    _writer = null;
    //clean up
    if (_rootDir.exists()) {
      FileUtils.forceDeleteOnExit(_rootDir);
    }
  }

  //-------------------------------------------------------------------------
  @Test
  public void ticksWriting() throws Exception {
    ZonedDateTime startTime = ZonedDateTime.now(Clock.systemUTC());
    
    //run test for 5secs
    long runTime = 5000;
    ExecutorService writerExecutor = Executors.newSingleThreadExecutor();
    Future<?> writerFuture = writerExecutor.submit(_writer);
    
    //create ticks generators
    ExecutorService ticksGeneratorExec = Executors.newSingleThreadExecutor();
    Future<?> ticksGenFuture = ticksGeneratorExec.submit(_ticksGenerator);
    
    s_logger.info("Test running for {}ms to generate ticks", runTime);
    Thread.sleep(runTime);
    
    //terminate ticks generation after 1mins
    _ticksGenerator.terminate();
    sendTerminateMessage();
    
    //test should fail if ticksGenerator throws an exception
    ticksGenFuture.get();
    ticksGeneratorExec.shutdown();
    ticksGeneratorExec.awaitTermination(1, TimeUnit.SECONDS);
    
    //test should fail if writer throws an exception
    writerFuture.get();
    writerExecutor.shutdown();
    writerExecutor.awaitTermination(1, TimeUnit.SECONDS);
    
    ZonedDateTime endTime = ZonedDateTime.now(Clock.systemUTC());
    
    //now lets replay generated allTicks.dat
    Set<String> buids = Sets.newHashSet(_ticker2buid.values());
    UnitTestTickReceiver receiver = new UnitTestTickReceiver();
    BloombergTicksReplayer player = new BloombergTicksReplayer(Mode.AS_FAST_AS_POSSIBLE, _rootDir.getAbsolutePath(), receiver, startTime, endTime, buids);
    player.start();
    while (player.isRunning()) {
      Thread.sleep(1000);
    }
    assertTrue(receiver.count() > 0);
  }

  @Test(invocationCount = 5, successPercentage = 19)
  public void performance() throws Exception {
    ExecutorService writerExecutor = Executors.newSingleThreadExecutor();
    Future<?> writerFuture = writerExecutor.submit(_writer);
    
    double nStartTime = System.currentTimeMillis ();
    
    //create ticks generators
    List<RandomTicksGeneratorJob> ticksGeneratorsList = new ArrayList<RandomTicksGeneratorJob>();
    List<Thread> ticksGeneratorThreads = new ArrayList<Thread>();
    for (int i = 0; i < TICKS_GENERATOR_THREAD_SIZE; i++) {
      RandomTicksGeneratorJob ticksGeneratorJob = new RandomTicksGeneratorJob(new ArrayList<String>(_ticker2buid.keySet()), _allTicksQueue);
      ticksGeneratorsList.add(ticksGeneratorJob);
      Thread thread = new Thread(ticksGeneratorJob, "TicksGenerator"+i);
      thread.start();
      ticksGeneratorThreads.add(thread);
    }
    
    s_logger.info("Test running for 1min to gather stats");
    Thread.sleep(RUN_DURATION);
    
    for (RandomTicksGeneratorJob ticksGeneratorJob : ticksGeneratorsList) {
      ticksGeneratorJob.terminate();
    }
    
    //wait for all ticksGenerator threads to finish
    for (Thread thread : ticksGeneratorThreads) {
      thread.join();
    }
    
    //send terminate message for tickWriter to terminate
    sendTerminateMessage();
    
    //test should fail if writer throws an exception
    writerFuture.get();
    writerExecutor.shutdown();
    writerExecutor.awaitTermination(1, TimeUnit.SECONDS);
    
    double nRunDuration = System.currentTimeMillis () - nStartTime;
    
    double nTicks = ((double)_writer.getNTicks()/nRunDuration) * 1000;
    s_logger.info("ticks {}/s", nTicks);
    double nWrites = ((double)_writer.getNWrites()/nRunDuration) * 1000;
    s_logger.info("fileOperations {}/s", nWrites);
    double nBlocks = (double)_writer.getNBlocks()/(double)_writer.getNWrites();
    s_logger.info("average blocks {}bytes", nBlocks);
    
    assertTrue("reportInterval > testRunTime", REPORT_INTERVAL > nRunDuration);
    if ((nWrites * nBlocks) < WRITER_SPEED_THRESHOLD) {
      s_logger.warn("BloombergTickWriter looks like running really slower than {}b/s", WRITER_SPEED_THRESHOLD);
    }
  }

  private void sendTerminateMessage() throws Exception {
    _allTicksQueue.put(BloombergTickReplayUtils.getTerminateMessage());
  }

  private class UnitTestTickReceiver implements BloombergTickReceiver {
    private Random _valueGenerator = new Random(RandomTicksGeneratorJob.RANDOM_SEED);
    private int _count;
    
    public void tickReceived(BloombergTick msg) {
      _count++;
      FudgeMsg randomStandardTick = BloombergTestUtils.makeRandomStandardTick(_valueGenerator, s_fudgeContext);
      FudgeMsg actual = msg.getFields();
      FudgeMsg expected = randomStandardTick.getMessage(FIELDS_KEY);
      assertAllFieldsMatch(expected, actual);
    }

    int count() {
      return _count;
    }
  }

  private static void assertAllFieldsMatch(FudgeMsg expectedMsg, FudgeMsg actualMsg) {
    FudgeUtils.assertAllFieldsMatch(expectedMsg, actualMsg, true);
  }

}