PhysicalSourceConfig.java

/*
 * $Id: PhysicalSourceConfig.java 272015 2011-05-21 03:03:57Z cbotev $
 */
package com.linkedin.databus2.relay.config;

import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.json.JSONObject;

import com.linkedin.databus.core.DbusEventBuffer;
import com.linkedin.databus.core.data_model.LogicalSource;
import com.linkedin.databus.core.data_model.PhysicalSource;
import com.linkedin.databus.core.util.ConfigBuilder;
import com.linkedin.databus.core.util.IdNamePair;
import com.linkedin.databus.core.util.InvalidConfigException;
import com.linkedin.databus2.core.BackoffTimerStaticConfigBuilder;
import com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig.ChunkingType;


/**
 * Configuration for a physical databus source - typically a database or another relay.
 * A single PhysicalSource can have multiple logical sources. The distinction being that a physical
 * has some sort of sequence generator (SCN if the source DB is Oracle) shared across all logical
 * sources.
 *
 * @author Jemiah Westerman<jwesterman@linkedin.com>
 * @version $Revision: 272015 $
 */
public class PhysicalSourceConfig implements ConfigBuilder<PhysicalSourceStaticConfig>
{

  public static final long DEFAULT_DBPOLL_INTERVAL_MILLIS = 500;
  public static final long MAX_DBPOLL_INTERVAL_MILLIS = 5 * 60 * 1000;

  // these default values are used for creating a default partition
  // for cases when no physical partition configuration is present (legacy code)
  public static final String DEFAULT_PHYSICAL_SOURCE_URI = "default_physical_source_uri";
  public static final Integer DEFAULT_PHYSICAL_PARTITION = 0;
  public static final String DEFAULT_PHYSICAL_PARTITION_NAME = "default_partition_name";
  public static final String DEFAULT_PHYSICAL_SOURCE_NAME = "default_physical_config_name";
  public static final String PHYSICAL_SOURCE_BASE_DIR = "databus.relay.base.string";
  public static final String PHYSICAL_SOURCE_BASE_DIR_DEFAULT = ".";

  public static final String DEFAULT_CHUNKING_TYPE = ChunkingType.NO_CHUNKING.toString();
  public static final long DEFAULT_SCN_CHUNK_SIZE = 20000;
  public static final long DEFAULT_TXN_CHUNK_SIZE = 20000;
  public static final long DEFAULT_SCN_CHUNKED_THRESHOLD = 20000;
  public static final long DEFAULT_MAX_SCN_DELAY_MS = 300000;
  public static final int DEFAULT_LARGEST_EVENT_SIZE = 1 * 1024*1024; //1MB
  public static final long DEFAULT_LARGEST_WINDOW_SIZE = 5*1024*1024; //5MB

  private String _name; // for example - database name
  private int _id;      // physical partition
  private String _uri;  // physical machne URI
  private String _resourceKey; // resource key which was used in case of Cluster manager
  private String _role;
  private long _slowSourceQueryThreshold;
  private long _restartScnOffset;
  private List<LogicalSourceConfig> _sources;
  private BackoffTimerStaticConfigBuilder _retries;
  private String _chunkingType;
  private long _txnsPerChunk;
  private long _scnChunkSize;
  private long _chunkedScnThreshold;
  private long _maxScnDelayMs;
  private  DbusEventBuffer.Config _dbusEventBuffer;
  // Used mainly by simulator
  // Used for GG relays to throttle rate at which events are read from trail files
  private long _eventRatePerSec;
  // Used for GG relays to specify the duration for which throttling be in effect after startup
  // When not specified, or equal to zero means that throttling is disabled
  private long _maxThrottleDurationInSecs;

 //used by chained relays to estimate , but could be used by regular relays for appropriate dynamic sizing or checks
  private int _largestEventSizeInBytes;
  private long _largestWindowSizeInBytes;

  //Golden gate relay specific configs
  /**
   *   If _errorOnMissingFields set to true, then the parser will terminate on missing fields in xmlTrail file,
   *   if set to false, then the parser will not terminate on missing fields (will use null value for the missing fields),
   *   provided it's NOT the primary key.
   */
  private boolean _errorOnMissingFields;
  /**
   * The config is used to specify what would be the xml version that the trail files are based on. The default value is 1.1
   * Accepts 1.0/1.1
   */
  private String _xmlVersion;
  /**
   * The type of encoding used by the xml, the default value for this config is ISO-8859-1
   */
  private String _xmlEncoding;

  /**
   * Config for deciding if an event is replicated or not !!
   */
  private ReplicationBitSetterConfig _replBitSetter;

  public PhysicalSourceConfig()
  {
    _id = 0;
    _resourceKey = "";
    _role = PhysicalSource.PHYSICAL_SOURCE_MASTER;
    _sources = new ArrayList<LogicalSourceConfig>();
    _restartScnOffset = 0;
    _slowSourceQueryThreshold = 3000;
    _retries = new BackoffTimerStaticConfigBuilder();
    _retries.setInitSleep(DEFAULT_DBPOLL_INTERVAL_MILLIS);
    _retries.setMaxRetryNum(-1);
    _retries.setMaxSleep(MAX_DBPOLL_INTERVAL_MILLIS);
    _retries.setSleepIncFactor(2.0);
    _retries.setSleepIncDelta(0);
    _chunkingType = DEFAULT_CHUNKING_TYPE;
    _txnsPerChunk = DEFAULT_TXN_CHUNK_SIZE;
    _scnChunkSize = DEFAULT_SCN_CHUNK_SIZE;
    _chunkedScnThreshold = DEFAULT_SCN_CHUNKED_THRESHOLD;
    _maxScnDelayMs = DEFAULT_MAX_SCN_DELAY_MS;
    _largestEventSizeInBytes = DEFAULT_LARGEST_EVENT_SIZE;
    _largestWindowSizeInBytes = DEFAULT_LARGEST_WINDOW_SIZE;
    _eventRatePerSec=10;
    _maxThrottleDurationInSecs = 0;
    _errorOnMissingFields = true;
    _dbusEventBuffer = null; //This buffer is per physical source, if not initialized in multi-tenant source config, the global eventbuffer config is used.
    _xmlEncoding = "ISO-8859-1";
    _replBitSetter = new ReplicationBitSetterConfig();
    _xmlVersion = "1.0";
  }

  /** create a PhysicalSourceConfiguration without any logical sources
   * use addSource(LogicalSourceConfig s) to add logical source
   */
  public PhysicalSourceConfig(String pSourceName, String pUri, int pPartionId) {
    this();

    setId(pPartionId);
    setName(pSourceName);
    setUri(pUri);
  }

  /** create physical source with some default values
   *  CAUTION - this createas a default physical partition (0)
   *  please use only in case when actual physical partition parameters are unavailable
   **/
  public PhysicalSourceConfig(Collection<IdNamePair> srcIds) {
    this();

    List<LogicalSourceConfig> newSources =
        new ArrayList<LogicalSourceConfig>(srcIds.size());

    for(IdNamePair p : srcIds) {
      LogicalSourceConfig source =  new LogicalSourceConfig();
      source.setId(p.getId().shortValue());
      source.setName(p.getName());
      // these two shouldn't ever be used
      source.setPartitionFunction("DefaultPartition");
      source.setUri("defaultUri");
      newSources.add(source);
    }
    setSources(newSources);
    setName("DefaultSource");
    setId(DEFAULT_PHYSICAL_PARTITION);
    setUri(DEFAULT_PHYSICAL_SOURCE_URI);
  }

  /** create physical source with some default values
   *  CAUTION - this createas a default physical partition (0)
   *  please use only in case when actual physical partition parameters are unavailable
   **/
  public static PhysicalSourceConfig createFromLogicalSources(Collection<LogicalSource> srcIds) {
    PhysicalSourceConfig res = new PhysicalSourceConfig();

    List<LogicalSourceConfig> newSources =
        new ArrayList<LogicalSourceConfig>(srcIds.size());

    for(LogicalSource p : srcIds) {
      LogicalSourceConfig source =  new LogicalSourceConfig();
      source.setId(p.getId().shortValue());
      source.setName(p.getName());
      // these two shouldn't ever be used
      source.setPartitionFunction("DefaultPartition");
      source.setUri("defaultUri");
      newSources.add(source);
    }
    res.setSources(newSources);
    res.setName(DEFAULT_PHYSICAL_PARTITION_NAME);
    res.setId(DEFAULT_PHYSICAL_PARTITION);
    res.setUri(DEFAULT_PHYSICAL_SOURCE_URI);

    return res;
  }

  /**
   * Check that none of the configuration settings are null.
   * @throws InvalidConfigException if one or more settings are null
   */
  public void checkForNulls()
      throws InvalidConfigException
      {
    if(_name == null || _name.length() == 0)
    {
      throw new InvalidConfigException("Name cannot be null or empty.");
    }

    if(_id < 0)
      throw new  InvalidConfigException("Physical source id cannot be empty.");

    if(_uri == null || _name.length() == 0)
    {
      throw new InvalidConfigException("URI cannot be null or empty.");
    }

    if(_sources == null || _sources.size() == 0)
    {
      throw new InvalidConfigException("Sources cannot be null or empty.");
    }

    for(LogicalSourceConfig source : _sources)
    {
      source.checkForNulls();
    }
    if (_slowSourceQueryThreshold < 0) {
      _slowSourceQueryThreshold = 0;
    }
      }

  public String getUri()
  {
    return _uri;
  }

  public void setUri(String uri)
  {
    _uri = uri;
  }

  public int getId()
  {
    return _id;
  }

  public void setId(int id)
  {
    _id = id;
  }

  public void setRole(String role) {
    _role = role;
  }

  public String getRole() {
    return _role;
  }

  public void setResourceKey(String rk) {
    _resourceKey = rk;
  }

  public String getResourceKey() {
    return _resourceKey;
  }

  private LogicalSourceConfig addOrGetSource(int index)
  {
    if (index >= _sources.size())
    {
      for (int i = _sources.size(); i <= index; ++i)
      {
        _sources.add(new LogicalSourceConfig());
      }
    }

    return _sources.get(index);
  }

  public LogicalSourceConfig getSource(int index)
  {
    LogicalSourceConfig result = addOrGetSource(index);
    return result;
  }

  public void setSource(int index, LogicalSourceConfig source)
  {
    addOrGetSource(index);
    _sources.set(index, source);
  }

  public void addSource(LogicalSourceConfig source) {
    if(! _sources.contains(source))
      _sources.add(source);
  }

  public List<LogicalSourceConfig> getSources()
  {
    return _sources;
  }

  public void setSources(List<LogicalSourceConfig> newSources)
  {
    _sources = newSources;
  }

  public String getName()
  {
    return _name;
  }

  public void setName(String name)
  {
    _name = name;
  }

  public long getSlowSourceQueryThreshold()
  {
    return _slowSourceQueryThreshold;
  }

  public void setSlowSourceQueryThreshold(long slowSourceQueryThreshold)
  {
    _slowSourceQueryThreshold = slowSourceQueryThreshold;
  }

  public long getRestartScnOffset() {
    return _restartScnOffset;
  }

  public void setRestartScnOffset(long r) {
    _restartScnOffset = r;
  }




  public int getLargestEventSizeInBytes() {
	  return _largestEventSizeInBytes;
  }

  public void setLargestEventSizeInBytes(int largestEventSizeInBytes) {
	  _largestEventSizeInBytes = largestEventSizeInBytes;
  }

  public long getLargestWindowSizeInBytes() {
	  return _largestWindowSizeInBytes;
  }

  public void setLargestWindowSizeInBytes(long largestWindowSizeInBytes) {
	  _largestWindowSizeInBytes = largestWindowSizeInBytes;
  }


  public boolean getErrorOnMissingFields()
  {
    return _errorOnMissingFields;
  }

  public void setErrorOnMissingFields(boolean errorOnMissingFields)
  {
    _errorOnMissingFields = errorOnMissingFields;
  }

  @Override
  public String toString()
  {
    try
    {
      ObjectMapper mapper = new ObjectMapper();
      StringWriter writer = new StringWriter();
      mapper.writeValue(writer, this);
      JSONObject jsonObj = new JSONObject(writer.toString());

      //The getMethod on this _dbusEventBuffer should be only called by the ConfigLoader
      if(_dbusEventBuffer!=null)
        {
          ObjectMapper mapperDbus = new ObjectMapper();
          StringWriter writerDbus = new StringWriter();
          mapperDbus.writeValue(writerDbus, _dbusEventBuffer);
          jsonObj.put("dbusEventBuffer", new JSONObject(writerDbus.toString()));
        }
      else
        jsonObj.put("dbusEventBuffer", JSONObject.NULL);

      return jsonObj.toString();
    }
    catch(Exception ex)
    {
      ex.printStackTrace();
      // Should never happen, but the ObjectMapper could throw an Exception.
      return super.toString();
    }
  }

  /**
   * Converse of toString; populate this object from the String
   * @throws IOException
   * @throws JsonMappingException
   * @throws JsonParseException
   */
  public static PhysicalSourceConfig fromString(String str) throws JsonParseException, JsonMappingException, IOException
  {
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig config = mapper.readValue(str,PhysicalSourceConfig.class);
    return config;
  }



  /**
   * Converse of toString; populate this object from a File handle pointing to a json file
   * @throws IOException
   * @throws JsonMappingException
   * @throws JsonParseException
   */
  public static PhysicalSourceConfig fromFile(File f) throws JsonParseException, JsonMappingException, IOException
  {
    ObjectMapper mapper = new ObjectMapper();
    PhysicalSourceConfig config = mapper.readValue(f,PhysicalSourceConfig.class);
    return config;
  }


  /**
   * Represents obj in map; rather obj->json->map
   * @return Map representation of object
   * @throws IOException
   * @throws JsonMappingException
   * @throws JsonParseException
   */
  public Map<String,Object> toMap() throws JsonParseException, JsonMappingException, IOException
  {
    String str = toString();
    ObjectMapper mapper = new ObjectMapper();
    HashMap<String,Object> map = mapper.readValue(str, HashMap.class);
    return map;
  }

  /**
   * Given a map of key value pairs; keys should correspond to field names ; return an object. Used to read configuration specification
   * @param map
   * @return
   * @throws IOException
   * @throws JsonMappingException
   * @throws JsonGenerationException
   */
  public static PhysicalSourceConfig fromMap(Map<String,Object> map) throws JsonGenerationException, JsonMappingException, IOException
  {

    ObjectMapper mapper = new ObjectMapper();
    StringWriter writer = new StringWriter();
    mapper.writeValue(writer, map);
    String str =  writer.toString();
    return fromString(str);
  }


  @Override
  public PhysicalSourceStaticConfig build() throws InvalidConfigException
  {
    checkForNulls();
    //check config options for chained relays
    if (_largestEventSizeInBytes >= _largestWindowSizeInBytes)
    {
      throw new InvalidConfigException("Invalid relay config: largestEventSizeInBytes has to be lesser than largestWindowSizeInBytes:"
          + " largestEventSizeInBytes=" + _largestEventSizeInBytes + " largestWindowSizeInBytes=" + _largestWindowSizeInBytes);
    }

    LogicalSourceStaticConfig[] sourcesStaticConfigs = new LogicalSourceStaticConfig[_sources.size()];
    for (int i = 0 ; i < _sources.size(); ++i)
    {
      sourcesStaticConfigs[i] = _sources.get(i).build();
    }
    ChunkingType chunkingType = ChunkingType.valueOf(_chunkingType);
    return new PhysicalSourceStaticConfig(_name, _id, _uri, _resourceKey,
                                          sourcesStaticConfigs, _role,
                                          _slowSourceQueryThreshold,
                                          _restartScnOffset,
                                          _retries.build(),
                                          chunkingType,
                                          _txnsPerChunk,
                                          _scnChunkSize,
                                          _chunkedScnThreshold,
                                          _maxScnDelayMs,
                                          _eventRatePerSec,
                                          _maxThrottleDurationInSecs,
                                          isDbusEventBufferSet()?_dbusEventBuffer.build():null,
                                          _largestEventSizeInBytes,
                                          _largestWindowSizeInBytes,
                                          _errorOnMissingFields,
                                          _xmlVersion,
                                          _xmlEncoding,
                                          _replBitSetter.build());
  }

  public BackoffTimerStaticConfigBuilder getRetries()
  {
    return _retries;
  }

  public void setRetries(BackoffTimerStaticConfigBuilder retries)
  {
    _retries = retries;
  }

  public String getChunkingType() {
	  return _chunkingType;
  }

  public void setChunkingType(String chunkingType) {
	  this._chunkingType = chunkingType;
  }

  public long getTxnsPerChunk() {
	  return _txnsPerChunk;
  }

  public void setTxnsPerChunk(long txnsPerChunk) {
	  this._txnsPerChunk = txnsPerChunk;
  }

  public long getScnChunkSize() {
	  return _scnChunkSize;
  }

  public void setScnChunkSize(long scnChunkSize) {
	  this._scnChunkSize = scnChunkSize;
  }

  public long getChunkedScnThreshold() {
	  return _chunkedScnThreshold;
  }

  public void setChunkedScnThreshold(long chunkedScnThreshold) {
	  this._chunkedScnThreshold = chunkedScnThreshold;
  }

  public long getMaxScnDelayMs() {
	return _maxScnDelayMs;
  }

  public void setMaxScnDelayMs(long maxScnDelayMs) {
	this._maxScnDelayMs = maxScnDelayMs;
  }

  public long getEventRatePerSec()
  {
	  return _eventRatePerSec;
  }

  public void setEventRatePerSec(long eventRatePerSec)
  {
	  _eventRatePerSec = eventRatePerSec;
  }

  public long getMaxThrottleDurationInSecs()
  {
    return _maxThrottleDurationInSecs;
  }

  public void setMaxThrottleDurationInSecs(long maxThrottleDurationInSecs)
  {
    _maxThrottleDurationInSecs = maxThrottleDurationInSecs;
  }

  @JsonIgnore
  public DbusEventBuffer.Config getDbusEventBuffer() {

    if(_dbusEventBuffer == null)
      _dbusEventBuffer = new DbusEventBuffer.Config();

    return _dbusEventBuffer;
  }

  public void setDbusEventBuffer(DbusEventBuffer.Config _dbusEventBuffer) {
    this._dbusEventBuffer = _dbusEventBuffer;
  }

  @JsonIgnore
  public boolean isDbusEventBufferSet()
  {
    return _dbusEventBuffer!=null?true:false;
  }

  public String getXmlVersion()
  {
    return _xmlVersion;
  }

  public void setXmlVersion(String xmlVersion)
  {
    _xmlVersion = xmlVersion;
  }

  public String getXmlEncoding()
  {
    return _xmlEncoding;
  }

  public void setXmlEncoding(String xmlEncoding)
  {
    _xmlEncoding = xmlEncoding;
  }

  public ReplicationBitSetterConfig getReplBitSetter()
  {
    return _replBitSetter;
  }

  public void setReplBitSetter(ReplicationBitSetterConfig replBitSetter)
  {
    this._replBitSetter = replBitSetter;
  }
}