DistributedLogReader.java

/*
 * Copyright © 2014-2016 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.logging.read;

import co.cask.cdap.common.conf.CConfiguration;
import co.cask.cdap.common.conf.Constants;
import co.cask.cdap.common.logging.LoggingContext;
import co.cask.cdap.logging.appender.kafka.StringPartitioner;
import co.cask.cdap.logging.filter.Filter;
import co.cask.cdap.logging.save.CheckpointManager;
import co.cask.cdap.logging.save.CheckpointManagerFactory;
import co.cask.cdap.logging.save.KafkaLogWriterPlugin;
import com.google.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Reads logs in a distributed setup, using kafka for latest logs and files for older logs.
 */
public final class DistributedLogReader implements LogReader {
  private static final Logger LOG = LoggerFactory.getLogger(DistributedLogReader.class);

  private final KafkaLogReader kafkaLogReader;
  private final FileLogReader fileLogReader;
  private final CheckpointManager checkpointManager;
  private final StringPartitioner partitioner;

  /**
   * Creates a DistributedLogReader object.
   */
  @Inject
  DistributedLogReader(CConfiguration cConf,
                              KafkaLogReader kafkaLogReader, FileLogReader fileLogReader,
                              CheckpointManagerFactory checkpointManagerFactory, StringPartitioner partitioner) {
    this.kafkaLogReader = kafkaLogReader;
    this.fileLogReader = fileLogReader;
    this.checkpointManager = checkpointManagerFactory.create(cConf.get(Constants.Logging.KAFKA_TOPIC),
                                                             KafkaLogWriterPlugin.CHECKPOINT_ROW_KEY_PREFIX);
    this.partitioner = partitioner;
  }

  @Override
  public void getLogNext(final LoggingContext loggingContext, final ReadRange readRange, final int maxEvents,
                              final Filter filter, final Callback callback) {
    // If latest logs are not requested, try reading from file.
    if (readRange != ReadRange.LATEST) {
      long checkpointTime = getCheckpointTime(loggingContext);
      // Read from file only if logs are saved for the loggingContext until fromTime
      if (readRange.getFromMillis() < checkpointTime) {
        fileLogReader.getLogNext(loggingContext, readRange, maxEvents, filter, callback);
        // If there are events from fileLogReader, return. Otherwise try in kafkaLogReader.
        if (callback.getCount() != 0) {
          LOG.trace("Got {} log entries from file", callback.getCount());
          return;
        }
      }
    }

    kafkaLogReader.getLogNext(loggingContext, readRange, maxEvents, filter, callback);
    LOG.trace("Got {} log entries from kafka", callback.getCount());

    // No logs in Kafka. This can happen for the latest run of a program, where the logs have been saved and
    // are expired in Kafka, but the checkpoint time is less than run end time - as this is the latest run.
    // In this case, return whatever you can find in saved logs.
    if (callback.getCount() == 0) {
      fileLogReader.getLogNext(loggingContext, readRange, maxEvents, filter, callback);
      LOG.trace("Got {} log entries from file", callback.getCount());
    }
  }

  @Override
  public void getLogPrev(final LoggingContext loggingContext, final ReadRange readRange, final int maxEvents,
                              final Filter filter, final Callback callback) {
    // If latest logs are not requested, try reading from file.
    if (readRange != ReadRange.LATEST) {
      long checkpointTime = getCheckpointTime(loggingContext);
      // Read from file only if logs are saved for the loggingContext until toTime
      if (readRange.getToMillis() < checkpointTime) {
        fileLogReader.getLogPrev(loggingContext, readRange, maxEvents, filter, callback);
        LOG.trace("Got {} log entries from file", callback.getCount());
        return;
      }
    }

    kafkaLogReader.getLogPrev(loggingContext, readRange, maxEvents, filter, callback);
    LOG.trace("Got {} log entries from kafka", callback.getCount());

    // No logs in Kafka. This can happen for the latest run of a program, where the logs have been saved and
    // are expired in Kafka, but the checkpoint time is less than run end time - as this is the latest run.
    // In this case, return whatever you can find in saved logs.
    if (callback.getCount() == 0) {
      fileLogReader.getLogPrev(loggingContext, readRange, maxEvents, filter, callback);
      LOG.trace("Got {} log entries from file", callback.getCount());
    }
  }

  @Override
  public void getLog(final LoggingContext loggingContext, final long fromTimeMs, final long toTimeMs,
                          final Filter filter, final Callback callback) {
    fileLogReader.getLog(loggingContext, fromTimeMs, toTimeMs, filter, callback);
  }

  private long getCheckpointTime(LoggingContext loggingContext) {
    int partition = partitioner.partition(loggingContext.getLogPartition(), -1);
    try {
      return checkpointManager.getCheckpoint(partition).getMaxEventTime();
    } catch (Exception e) {
      LOG.error("Got exception while reading checkpoint", e);
    }
    return -1;
  }
}