/**
 * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.linkedin.pinot.core.realtime.impl.kafka;

import com.linkedin.pinot.common.metrics.ServerMeter;
import com.linkedin.pinot.common.metrics.ServerMetrics;
import com.yammer.metrics.core.Meter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.linkedin.pinot.core.data.GenericRow;
import com.linkedin.pinot.core.realtime.StreamProvider;
import com.linkedin.pinot.core.realtime.StreamProviderConfig;
import kafka.consumer.ConsumerIterator;
import kafka.javaapi.consumer.ConsumerConnector;


/**
 *
 */
public class KafkaHighLevelConsumerStreamProvider implements StreamProvider {
  private static final Logger STATIC_LOGGER = LoggerFactory.getLogger(KafkaHighLevelConsumerStreamProvider.class);

  private KafkaHighLevelStreamProviderConfig streamProviderConfig;
  private KafkaMessageDecoder decoder;

  private ConsumerConnector consumer;
  private ConsumerIterator<byte[], byte[]> kafkaIterator;
  private long lastLogTime = 0;
  private long lastCount = 0;
  private ConsumerAndIterator consumerAndIterator;

  private Logger INSTANCE_LOGGER = STATIC_LOGGER;

  private ServerMetrics serverMetrics;
  private String tableAndStreamName;
  private long currentCount = 0L;

  private Meter tableAndStreamRowsConsumed = null;
  private Meter tableRowsConsumed = null;

  @Override
  public void init(StreamProviderConfig streamProviderConfig, String tableName, ServerMetrics serverMetrics)
      throws Exception {
    this.streamProviderConfig = (KafkaHighLevelStreamProviderConfig) streamProviderConfig;
    this.decoder = this.streamProviderConfig.getDecoder();
    tableAndStreamName = tableName + "-" + streamProviderConfig.getStreamName();
    INSTANCE_LOGGER = LoggerFactory.getLogger(
        KafkaHighLevelConsumerStreamProvider.class.getName() + "_" + tableName + "_" + streamProviderConfig
            .getStreamName());
    this.serverMetrics = serverMetrics;
  }

  @Override
  public void start() throws Exception {
    consumerAndIterator = KafkaConsumerManager.acquireConsumerAndIteratorForConfig(streamProviderConfig);
    kafkaIterator = consumerAndIterator.getIterator();
    consumer = consumerAndIterator.getConsumer();
  }

  @Override
  public void setOffset(long offset) {
    throw new UnsupportedOperationException();
  }

  @Override
  public GenericRow next(GenericRow destination) {
    if (kafkaIterator.hasNext()) {
      try {
        destination = decoder.decode(kafkaIterator.next().message(), destination);
        tableAndStreamRowsConsumed = serverMetrics.addMeteredTableValue(tableAndStreamName, ServerMeter.REALTIME_ROWS_CONSUMED, 1L, tableAndStreamRowsConsumed);
        tableRowsConsumed = serverMetrics.addMeteredGlobalValue(ServerMeter.REALTIME_ROWS_CONSUMED, 1L, tableRowsConsumed);
        ++currentCount;

        final long now = System.currentTimeMillis();
        // Log every minute or 100k events
        if (now - lastLogTime > 60000 || currentCount - lastCount >= 100000) {
          if (lastCount == 0) {
            INSTANCE_LOGGER.info("Consumed {} events from kafka stream {}", currentCount,
                this.streamProviderConfig.getStreamName());
          } else {
            INSTANCE_LOGGER.info("Consumed {} events from kafka stream {} (rate:{}/s)", currentCount - lastCount,
                this.streamProviderConfig.getStreamName(),
                (float) (currentCount - lastCount) * 1000 / (now - lastLogTime));
          }
          lastCount = currentCount;
          lastLogTime = now;
        }
        return destination;
      } catch (Exception e) {
        INSTANCE_LOGGER.warn("Caught exception while consuming events", e);
        serverMetrics.addMeteredTableValue(tableAndStreamName, ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS, 1L);
        serverMetrics.addMeteredGlobalValue(ServerMeter.REALTIME_CONSUMPTION_EXCEPTIONS, 1L);
        throw e;
      }
    }
    return null;
  }

  @Override
  public GenericRow next(long offset) {
    throw new UnsupportedOperationException();
  }

  @Override
  public long currentOffset() {
    throw new UnsupportedOperationException();
  }

  @Override
  public void commit() {
    consumer.commitOffsets();
    serverMetrics.addMeteredTableValue(tableAndStreamName, ServerMeter.REALTIME_OFFSET_COMMITS, 1L);
    serverMetrics.addMeteredGlobalValue(ServerMeter.REALTIME_OFFSET_COMMITS, 1L);
  }

  @Override
  public void commit(long offset) {
    throw new UnsupportedOperationException();
  }

  @Override
  public void shutdown() throws Exception {
    if (consumerAndIterator != null) {
      kafkaIterator = null;
      consumer = null;

      KafkaConsumerManager.releaseConsumerAndIterator(consumerAndIterator);
      consumerAndIterator = null;
    }
  }
}