RedisLKVSnapshotter.java

/**
 * Copyright (C) 2012 - present by OpenGamma Inc. and the OpenGamma group of companies
 * 
 * Please see distribution for license.
 */
package com.opengamma.integration.timeseries.snapshot;

import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.Pipeline;
import redis.clients.jedis.Response;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.opengamma.id.ExternalId;
import com.opengamma.util.ArgumentChecker;
import com.opengamma.util.monitor.OperationTimer;
import com.opengamma.util.redis.RedisConnector;

/**
 * Snapshot the last known values for all fields in Redis.
 */
public class RedisLKVSnapshotter {
  
  private static final Logger s_logger = LoggerFactory.getLogger(RedisLKVSnapshotter.class);
  
  private final Map<String, Boolean> _dataFieldBlackList = Maps.newHashMap();
  private final Map<String, Boolean> _schemeBlackList = Maps.newHashMap();
  private final String _normalizationRuleSetId;
  private final String _globalPrefix;
  private final RedisConnector _redisConnector;
  
  public RedisLKVSnapshotter(final BlackList dataFieldBlackList, final BlackList schemeBlackList, final String normalizationRuleSetId, final String globalPrefix, final RedisConnector redisConnector) {
    ArgumentChecker.notNull(normalizationRuleSetId, "normalizationRuleSetId");
    ArgumentChecker.notNull(globalPrefix, "globalPrefix");
    ArgumentChecker.notNull(redisConnector, "redisConnector");
    ArgumentChecker.notNull(dataFieldBlackList, "data field black list");
    ArgumentChecker.notNull(schemeBlackList, "scheme black list");
    
    _normalizationRuleSetId = normalizationRuleSetId;
    _globalPrefix = globalPrefix;
    _redisConnector = redisConnector;
    for (String dataField : dataFieldBlackList.getBlackList()) {
      _dataFieldBlackList.put(dataField.toUpperCase(), Boolean.TRUE);
    }
    for (String scheme : schemeBlackList.getBlackList()) {
      _schemeBlackList.put(scheme.toUpperCase(), Boolean.TRUE);
    }
    
  }
  
  /**
   * Gets the normalizationRuleSetId.
   * @return the normalizationRuleSetId
   */
  public String getNormalizationRuleSetId() {
    return _normalizationRuleSetId;
  }

  /**
   * Gets the globalPrefix.
   * @return the globalPrefix
   */
  public String getGlobalPrefix() {
    return _globalPrefix;
  }

  /**
   * Gets the redisConnector.
   * @return the redisConnector
   */
  public RedisConnector getRedisConnector() {
    return _redisConnector;
  }

  public Map<ExternalId, Map<String, String>> getLastKnownValues() {
    s_logger.debug("Reading Redis LKV values for normalizationRuleSetId:{} globalPrefix:{} dataFieldBlackList:{} schemeBlackList:{}", 
        new Object[] {getNormalizationRuleSetId(), getGlobalPrefix(), _dataFieldBlackList.keySet(), _schemeBlackList.keySet()});
    List<ExternalId> allSecurities = getAllSecurities();
    OperationTimer timer = new OperationTimer(s_logger, "Reading LKV for {} securities", allSecurities.size());
    Map<ExternalId, Map<String, String>> result = getLastKnownValues(allSecurities);
    timer.finished();
    return result;
  }
  
  @SuppressWarnings("unchecked")
  public Map<ExternalId, Map<String, String>> getLastKnownValues(final List<ExternalId> securities) {
    Map<ExternalId, Map<String, String>> result = Maps.newHashMap();
    JedisPool jedisPool = _redisConnector.getJedisPool();
    Jedis jedis = jedisPool.getResource();
    Pipeline pipeline = jedis.pipelined();
    //start transaction
    pipeline.multi();
    for (ExternalId identifier : securities) {
      String redisKey = generateRedisKey(identifier.getScheme().getName(), identifier.getValue(), getNormalizationRuleSetId());
      pipeline.hgetAll(redisKey);
    }
    Response<List<Object>> response = pipeline.exec();
    pipeline.sync();
    
    final Iterator<ExternalId> allSecItr = securities.iterator();
    final Iterator<Object> responseItr = response.get().iterator();
    while (responseItr.hasNext() && allSecItr.hasNext()) {
      result.put(allSecItr.next(), filterBlackListedTicks((Map<String, String>) responseItr.next()));
    }
    jedisPool.returnResource(jedis);
    return result;
  }
  
  private Map<String, String> filterBlackListedTicks(Map<String, String> ticks) {
    Map<String, String> result = Maps.newHashMap();
    for (Entry<String, String> entry : ticks.entrySet()) {
      final String fieldName = entry.getKey();
      if (_dataFieldBlackList.containsKey(fieldName.toUpperCase())) {
        continue;
      }
      result.put(entry.getKey(), entry.getValue());
    }
    return result;
  }

  private String generateRedisKey(String scheme, String identifier, String normalizationRuleSetId) {
    StringBuilder sb = new StringBuilder();
    if (getGlobalPrefix() != null) {
      sb.append(getGlobalPrefix());
    }
    sb.append(scheme);
    sb.append("-");
    sb.append(identifier);
    sb.append("[");
    sb.append(normalizationRuleSetId);
    sb.append("]");
    return sb.toString();
  }
  
  private List<ExternalId> getAllSecurities() {
    List<ExternalId> securities = Lists.newArrayList();
    Set<String> allSchemes = getAllSchemes();
    for (String scheme : allSchemes) {
      if (_schemeBlackList.containsKey(scheme.toUpperCase())) {
        continue;
      }
      Set<String> allIdentifiers = getAllIdentifiers(scheme);
      for (String identifier : allIdentifiers) {
        securities.add(ExternalId.of(scheme, identifier));
      }
    }
    return securities;
  }
  
  private Set<String> getAllSchemes() {
    JedisPool jedisPool = _redisConnector.getJedisPool();
    Jedis jedis = jedisPool.getResource();
    Set<String> allMembers = jedis.smembers(generateAllSchemesKey());
    jedisPool.returnResource(jedis);
    s_logger.info("Loaded {} schemes from Jedis (full contents in Debug level log)", allMembers.size());
    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Loaded schemes from Jedis: {}", allMembers);
    }
    return allMembers;
  }
  
  private String generateAllSchemesKey() {
    StringBuilder sb = new StringBuilder();
    if (getGlobalPrefix() != null) {
      sb.append(getGlobalPrefix());
    }
    sb.append("-<ALL_SCHEMES>");
    s_logger.debug("AllSchemeKey: {}", sb.toString());
    return sb.toString();
  }
  
  private String generatePerSchemeKey(String scheme) {
    StringBuilder sb = new StringBuilder();
    if (getGlobalPrefix() != null) {
      sb.append(getGlobalPrefix());
    }
    sb.append(scheme);
    sb.append("-");
    sb.append("<ALL_IDENTIFIERS>");
    s_logger.debug("PerSchemeKey: {}", sb.toString());
    return sb.toString();
  }
  
  private Set<String> getAllIdentifiers(String identifierScheme) {
    JedisPool jedisPool = _redisConnector.getJedisPool();
    Jedis jedis = jedisPool.getResource();
    Set<String> allMembers = jedis.smembers(generatePerSchemeKey(identifierScheme));
    jedisPool.returnResource(jedis);
    s_logger.info("Loaded {} identifiers from Jedis (full contents in Debug level log)", allMembers.size());
    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Loaded identifiers from Jedis: {}", allMembers);
    }
    return allMembers;
  }

}