DatabusBootstrapProducer.java

package com.linkedin.databus.bootstrap.producer;

/*
 *
 * Copyright 2013 LinkedIn Corp. All rights reserved
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.log4j.Logger;

import com.linkedin.databus.bootstrap.api.BootstrapProducerStatus;
import com.linkedin.databus.bootstrap.common.BootstrapConn;
import com.linkedin.databus.bootstrap.common.BootstrapDBCleaner;
import com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO;
import com.linkedin.databus.bootstrap.common.BootstrapProducerStatsCollector;
import com.linkedin.databus.client.DatabusHttpClientImpl;
import com.linkedin.databus.client.pub.CheckpointPersistenceProvider;
import com.linkedin.databus.client.pub.DatabusClientException;
import com.linkedin.databus.client.pub.ServerInfo;
import com.linkedin.databus.core.Checkpoint;
import com.linkedin.databus.core.DatabusThreadBase;
import com.linkedin.databus.core.DbusClientMode;
import com.linkedin.databus.core.data_model.DatabusSubscription;
import com.linkedin.databus.core.monitoring.mbean.StatsCollectors;
import com.linkedin.databus.core.util.ConfigLoader;
import com.linkedin.databus.core.util.InvalidConfigException;
import com.linkedin.databus2.core.DatabusException;
import com.linkedin.databus2.core.container.netty.ServerContainer;
import com.linkedin.databus2.core.container.request.BootstrapDBException;
import com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException;

public class DatabusBootstrapProducer extends DatabusHttpClientImpl implements
    BootstrapProducerCallback.ErrorCaseHandler
{
  public static final String MODULE = DatabusBootstrapProducer.class.getName();
  public static final Logger LOG = Logger.getLogger(MODULE);

  private final HashSet<SourceInfo> _registeredPhysicalSources;
  private final List<String> _registeredSources;

  private final Map<String, DatabusThreadBase> _applierThreads;
  private final BootstrapDBPeriodicTriggerThread _dbPeriodicTriggerThread;
  private final BootstrapDBDiskSpaceTriggerThread _dbDiskSpaceTriggerThread;
  private final BootstrapDBCleaner _dbCleaner;

  private final BootstrapDBMetaDataDAO _dbDao;
  private final Map<String, Integer> _srcNameIdMap;

  protected final StatsCollectors<BootstrapProducerStatsCollector> _bootstrapProducerStatsCollectors;
  private final BootstrapProducerStatsCollector _applierStatsCollector;
  private final BootstrapProducerStaticConfig _bootstrapProducerStaticConfig;

  /**
   * @param config
   * @throws SQLException
   * @throws ClassNotFoundException
   * @throws IllegalAccessException
   * @throws InstantiationException
   * @throws DatabusClientException
   * @throws DatabusException
   * @throws BootstrapDatabaseTooOldException
   * @throws Exception
   */

  public DatabusBootstrapProducer(BootstrapProducerConfig config)
      throws IOException, InvalidConfigException, InstantiationException,
      IllegalAccessException, ClassNotFoundException, SQLException,
      DatabusClientException, DatabusException, BootstrapDBException
  {
    this(config.build());
  }

  public DatabusBootstrapProducer(
      BootstrapProducerStaticConfig bootstrapProducerStaticConfig)
      throws IOException, InvalidConfigException, InstantiationException,
      IllegalAccessException, ClassNotFoundException, SQLException,
      DatabusClientException, DatabusException, BootstrapDBException
  {
    super(bootstrapProducerStaticConfig.getClient());
    _registeredPhysicalSources = new HashSet<SourceInfo>();
    decouplePhysicalSources();
    _bootstrapProducerStaticConfig = bootstrapProducerStaticConfig;
    _registeredSources = new ArrayList<String>();
    _applierStatsCollector = new BootstrapProducerStatsCollector(
        getContainerStaticConfig().getId(), "bootstrapApplier", true, true,
        getMbeanServer(), null);
    _bootstrapProducerStatsCollectors = new StatsCollectors<BootstrapProducerStatsCollector>();

    _applierThreads = new HashMap<String, DatabusThreadBase>();

    BootstrapConn conn = new BootstrapConn();
    final boolean autoCommit = false;
    _dbDao = new BootstrapDBMetaDataDAO(conn,
        bootstrapProducerStaticConfig.getBootstrapDBHostname(),
        bootstrapProducerStaticConfig.getBootstrapDBUsername(),
        bootstrapProducerStaticConfig.getBootstrapDBPassword(),
        bootstrapProducerStaticConfig.getBootstrapDBName(), autoCommit);
    _srcNameIdMap = new HashMap<String, Integer>();
    conn.initBootstrapConn(autoCommit,
        bootstrapProducerStaticConfig.getBootstrapDBUsername(),
        bootstrapProducerStaticConfig.getBootstrapDBPassword(),
        bootstrapProducerStaticConfig.getBootstrapDBHostname(),
        bootstrapProducerStaticConfig.getBootstrapDBName());
    if (!_dbDao.doesMinScnTableExist())
    {
      LOG.error("Bootstrap table not found! Please create table: "
          + BootstrapDBMetaDataDAO.CREATE_MINSCN_TABLE);
      throw new BootstrapDBException(
          "Bootstrap DB does not have necessary meta data table! "
              + BootstrapDBMetaDataDAO.MIN_SCN_TABLE_NAME);
    }
    initBootstrapDBMetadata();

    // callback should only be registered after DBMetadata is initialized.
    LOG.info("The Bootstrap Producer is configured for "
        + _registeredPhysicalSources.size() + " sources");
    for (SourceInfo sourceInfo : _registeredPhysicalSources)
    {
      LOG.info("Creating BootstrapProducer callback for PhysicalSource: "
          + sourceInfo.getPhysicalSourceName());
      BootstrapProducerStatsCollector producerStatsCollector = new BootstrapProducerStatsCollector(
          getContainerStaticConfig().getId(),
          sourceInfo.getPhysicalSourceName(), true, true, getMbeanServer(),
          sourceInfo.getLogicalSources());
      _bootstrapProducerStatsCollectors.addStatsCollector(
          sourceInfo.getPhysicalSourceName(), producerStatsCollector);
      registerProducerCallback(sourceInfo.getLogicalSources(),
          producerStatsCollector);
    }
    validateAndRepairBootstrapDBCheckpoint();

    for (String source: _registeredSources)
    {
      LOG.info("Creating ApplierThread for source = " + source);
      final String name = source + "BootstrapApplier";
      BootstrapApplierThread applierThread = new BootstrapApplierThread(name, source,
          _bootstrapProducerStaticConfig, _applierStatsCollector);
      _applierThreads.put(source, applierThread);
    }

    // Create BootstrapDBCleaner
    final String dbCleanerName = "DBCleaner";
    _dbCleaner = new BootstrapDBCleaner(dbCleanerName,
        _bootstrapProducerStaticConfig.getCleaner(),
        _bootstrapProducerStaticConfig, _applierThreads, _registeredSources);

    // Create periodic trigger thread
    _dbPeriodicTriggerThread = new BootstrapDBPeriodicTriggerThread(_dbCleaner,
        _bootstrapProducerStaticConfig.getCleaner().getPeriodSpaceTrigger());

    // Create disk space cleaner thread
    _dbDiskSpaceTriggerThread = new BootstrapDBDiskSpaceTriggerThread(_dbCleaner,
        _bootstrapProducerStaticConfig.getCleaner().getDiskSpaceTrigger());

  }

  /**
   * The method helps identify the Physical sources that are listed in the
   * bootstrap producer config (Which is really the client config) The Physical
   * sources are identified by comparing the list of subscriptions/logical
   * sources. There are two acceptable cases: 1. Two relays provide different
   * logical sources -> Two different physical sources 2. Two relays provide the
   * same logical sources (for load balancing/fault tolerance) -> Only one
   * physical source (The multitenant-client library underneath is aware that
   * two relays provide the same logical sources)
   *
   * @return A set of Physical sources which each contains a list of logical
   *         sources.
   *
   */
  private void decouplePhysicalSources()
  {
    DatabusHttpClientImpl.RuntimeConfig clientRtConfig = getClientConfigManager()
        .getReadOnlyConfig();

    for (ServerInfo relayInfo : clientRtConfig.getRelays())
    {

      if (relayInfo == null || relayInfo.getSources() == null)
        LOG.error("No sources specified in the client config for the bootstrap producer");

      if (relayInfo.getPhysicalSourceName() == null)
      {
        LOG.error("PhysicalSource name not specified");
      }

      SourceInfo sourceInfo = new SourceInfo(relayInfo.getPhysicalSourceName(),
          relayInfo.getSources());
      _registeredPhysicalSources.add(sourceInfo);
    }
  }

  /**
   *
   *
   * Compares the Checkpoint and bootstrap_producer_state's SCN to check against
   * the possibility of gap in event consumption!! If gap is found, makes
   * best-effort to repair it.
   *
   * Three Scenario that should be allowed
   *
   * 1. Bootstrap Producer started after seeding In this case checkpoint SCN
   * should not be greater than producer SCN. 2. Bootstrap Producer started
   * after adding sources without seeding In this case the producer SCN is "-1".
   * The checkpoint "could" be empty 3. Bootstrap Producer bounced In this case
   * checkpoint SCN should not be greater than producer SCN.
   *
   * @throws SQLException
   * @throws BootstrapDBException
   *           if there is a gap and cannot be repaired
   * @throws IOException
   *           `
   */
  private void validateAndRepairBootstrapDBCheckpoint() throws SQLException,
      BootstrapDBException, IOException
  {
    LOG.info("Validating bootstrap DB checkpoints !!");

    for (List<DatabusSubscription> subsList : _relayGroups.keySet())
    {
      List<String> sourceNames = DatabusSubscription.getStrList(subsList);
      long scn = -1;
      try
      {
        scn = _dbDao.getMinWindowSCNFromStateTable(sourceNames,
            "bootstrap_producer_state");
      } catch (BootstrapDBException ex)
      {
        LOG.error(
            "Got exception while trying to fetch SCN from bootstrap_producer_state for sources :"
                + sourceNames, ex);
        throw ex;
      }

      CheckpointPersistenceProvider provider = getCheckpointPersistenceProvider();
      Checkpoint cp = provider.loadCheckpoint(sourceNames);
      LOG.info("Bootstrap Producer SCN :" + scn + ", Checkpoint :" + cp);

      if (null != cp)
      {
        if (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION)
        {
          // Bootstrapping bootstrap Producer not yet supported !!
          String msg = "Bootstrap Producer starting from non-online consumption mode for sources :"
              + sourceNames + ", Ckpt :" + cp;
          LOG.error(msg);
          throw new BootstrapDBException(msg);
        }
        else
        {
          String msg = null;
          if (((cp.getWindowScn() > scn) && (scn > -1))
              || ((cp.getWindowScn() < scn) && (scn > -1)))
          {

            if (((cp.getWindowScn() > scn) && (scn > -1)))
              LOG.warn("Non-Empty checkpint. Bootstrap Producer is at SCN:"
                  + scn
                  + ", while checkpoint is :"
                  + cp
                  + ", Could result in gap in event consumption. Repairing ckpt !!");
            else
              LOG.info("Non-Empty checkpoint. Bootstrap Producer is at SCN:"
                  + scn + ", while checkpoint is :" + cp
                  + ", Copying producer Scn to checkpoint !!");

            cp.setWindowScn(scn);
            cp.setWindowOffset(-1);
            try
            {
              provider.removeCheckpoint(sourceNames);
              provider.storeCheckpoint(sourceNames, cp);

              // Check if persisted properly
              cp = provider.loadCheckpoint(sourceNames);
              if ((null == cp)
                  || (cp.getWindowScn() != scn)
                  || (cp.getWindowOffset() != -1)
                  || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION))
              {
                msg = "Unable to repair and store the new checkpoint (" + cp
                    + ") to make it same as producer SCN (" + scn + ") !!";
                LOG.fatal(msg);
                throw new BootstrapDBException(msg);
              }
            } catch (IOException ex)
            {
              msg = "Unable to repair and store the new checkpoint (" + cp
                  + ") to make it same as producer SCN (" + scn + ") !!";
              LOG.fatal(msg, ex);
              throw new BootstrapDBException(msg);
            }
          }
        }
      }
      else
      {
        /**
         * Currently since bootstrapping is not available, a null ckpt would
         * result in flexible checkpoint and could result in gap !!
         */
        if (scn > -1)
        {
          String msg = "Empty checkpoint. Bootstrap Producer SCN is at SCN:"
              + scn
              + ", while checkpoint is null !! Could result in gap in event consumption. Repairing ckpt !!";
          LOG.warn(msg);
          cp = new Checkpoint();
          cp.setWindowScn(scn);
          cp.setWindowOffset(-1);
          cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
          try
          {
            provider.removeCheckpoint(sourceNames);
            provider.storeCheckpoint(sourceNames, cp);

            // Check if persisted properly
            cp = provider.loadCheckpoint(sourceNames);
            if ((null == cp)
                || (cp.getWindowScn() != scn)
                || (cp.getWindowOffset() != -1)
                || (cp.getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION))
            {
              LOG.fatal("Unable to repair and store the checkpoint (" + cp
                  + ") to make it same as producer SCN (" + scn + ") !!");
              throw new BootstrapDBException(msg);
            }
          } catch (IOException ex)
          {
            msg = "Unable to repair and store the checkpoint (" + cp
                + ") to make it same as producer SCN (" + scn + ") !!";
            LOG.fatal(msg, ex);
            throw new BootstrapDBException(msg);
          }
        }
      }
    }
    LOG.info("Validating bootstrap DB checkpoints done successfully!!");
  }

  private void registerProducerCallback(List<String> logicalSourceList,
      BootstrapProducerStatsCollector statsCollector)
      throws SQLException, DatabusClientException, DatabusException
  {

    // create callback for producer to populate data into log_* tables
    BootstrapProducerCallback bootstrapCallback = new BootstrapProducerCallback(
        _bootstrapProducerStaticConfig, statsCollector, this, logicalSourceList);
    registerDatabusStreamListener(bootstrapCallback, logicalSourceList, null);
  }

  private void initBootstrapDBMetadata() throws SQLException,
      BootstrapDatabaseTooOldException
  {
    DatabusHttpClientImpl.RuntimeConfig clientRtConfig = getClientConfigManager()
        .getReadOnlyConfig();

    // create source list
    for (ServerInfo relayInfo : clientRtConfig.getRelays())
    {
      _registeredSources.addAll(relayInfo.getSources());
      for (String source : _registeredSources)
      {
        BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatus = _dbDao
            .getSrcIdStatusFromDB(source, false);

        if (0 > srcIdStatus.getSrcId())
        {
          int newState = BootstrapProducerStatus.NEW;

          if (!_bootstrapProducerStaticConfig.isBootstrapDBStateCheck())
          {
            // TO allow test framework to listen to relay directly,DBStateCheck
            // flag is used
            newState = BootstrapProducerStatus.ACTIVE;
          }
          _dbDao.addNewSourceInDB(source, newState);
        }

        srcIdStatus = _dbDao.getSrcIdStatusFromDB(source, false);
        _srcNameIdMap.put(source, srcIdStatus.getSrcId());

        if (_bootstrapProducerStaticConfig.isBootstrapDBStateCheck())
        {
          if (!BootstrapProducerStatus.isReadyForConsumption(srcIdStatus
              .getStatus()))
            throw new BootstrapDatabaseTooOldException(
                "Bootstrap DB is not ready to read from relay !! Status :"
                    + srcIdStatus);
        }
      }
    }
  }

  public List<String> getRegisteredSources()
  {
    return _registeredSources;
  }

  @Override
  public void doStart()
  {
    super.doStart();

    if (_bootstrapProducerStaticConfig.getRunApplierThreadOnStart())
    {
      for (Map.Entry<String, DatabusThreadBase> applierThreadEntry: _applierThreads.entrySet())
      {
        LOG.info("Starting applier thread for source = " + applierThreadEntry.getValue());
        applierThreadEntry.getValue().start();
      }
    }
    else
    {
      LOG.info("Not starting any applier threads because the config getRunApplierThreadOnStart is false");
    }

    if (_bootstrapProducerStaticConfig.getCleaner().getDiskSpaceTrigger().isEnable())
    {
      LOG.info("Starting disk space trigger thread");
      _dbDiskSpaceTriggerThread.start();
    }

    if (_bootstrapProducerStaticConfig.getCleaner().getPeriodSpaceTrigger().isEnable())
    {
      LOG.info("Starting periodic trigger thread");
      _dbPeriodicTriggerThread.start();
    }

    LOG.info(DatabusBootstrapProducer.class.getName() + " is running ...");
  }

  public static void main(String[] args) throws Exception
  {
    // Properties startupProps = BootstrapConfig.loadConfigProperties(args);
    Properties startupProps = ServerContainer.processCommandLineArgs(args);

    BootstrapProducerConfig producerConfig = new BootstrapProducerConfig();
    ConfigLoader<BootstrapProducerStaticConfig> staticProducerConfigLoader = new ConfigLoader<BootstrapProducerStaticConfig>(
        "databus.bootstrap.", producerConfig);
    BootstrapProducerStaticConfig staticProducerConfig = staticProducerConfigLoader
        .loadConfig(startupProps);

    DatabusBootstrapProducer bootstrapProducer = new DatabusBootstrapProducer(
        staticProducerConfig);
    bootstrapProducer.registerShutdownHook();
    bootstrapProducer.startAndBlock();
  }

  @Override
  protected void doShutdown()
  {
    super.doShutdown();
    for (Map.Entry<String, DatabusThreadBase> applierThreadEntry: _applierThreads.entrySet())
    {
      DatabusThreadBase applierThread = applierThreadEntry.getValue();
      if (applierThread.isAlive())
      {
        applierThread.shutdownAsynchronously();
        applierThread.interrupt();
        applierThread.awaitShutdownUniteruptibly();
      }
    }

    if (_dbDiskSpaceTriggerThread.isAlive())
    {
      _dbDiskSpaceTriggerThread.shutdownAsynchronously();
      _dbDiskSpaceTriggerThread.interrupt();
      _dbDiskSpaceTriggerThread.awaitShutdownUniteruptibly();
    }

    if (_dbPeriodicTriggerThread.isAlive())
    {
      _dbPeriodicTriggerThread.shutdownAsynchronously();
      _dbPeriodicTriggerThread.interrupt();
      _dbPeriodicTriggerThread.awaitShutdownUniteruptibly();
    }


  }

  public StatsCollectors<BootstrapProducerStatsCollector> getProducerStatsCollectors()
  {
    return _bootstrapProducerStatsCollectors;
  }

  public BootstrapProducerStatsCollector getApplierStatsCollector()
  {
    return _applierStatsCollector;
  }

  @Override
  public void onErrorRetryLimitExceeded(String message, Throwable exception)
  {
    LOG.fatal("Error Retry Limit reached. Message :(" + message
        + "). Stopping Bootstrap Producer Service. Exception Received :",
        exception);
    doShutdown();
  }

  /*
   * This class holds information about the physical source and the list of
   * logical sources that belong to them
   */
  static private class SourceInfo
  {
    private String PhysicalSourceName;
    private List<String> logicalSources;

    public List<String> getLogicalSources()
    {
      return logicalSources;
    }

    public void setLogicalSources(List<String> logicalSources)
    {
      this.logicalSources = logicalSources;
    }

    public String getPhysicalSourceName()
    {
      return PhysicalSourceName;
    }

    public void setPhysicalSourceName(String physicalSourceName)
    {
      PhysicalSourceName = physicalSourceName;
    }

    private SourceInfo(String physicalSourceName, List<String> logicalSources)
    {
      PhysicalSourceName = physicalSourceName;
      this.logicalSources = logicalSources;
    }

    @Override
    public boolean equals(Object obj)
    {

      if (null == obj)
        return false;
      if (!(obj instanceof SourceInfo))
        return false;
      SourceInfo other = (SourceInfo) obj;
      return this.PhysicalSourceName.equals(other.getPhysicalSourceName());
    }

    @Override
    public int hashCode()
    {
      return PhysicalSourceName != null ? PhysicalSourceName.hashCode() : 0;
    }
  }
}