AbstractKafkaLogProcessor.java

/*
 * Copyright © 2015 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.logging.save;

import co.cask.cdap.logging.kafka.KafkaLogEvent;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;

import java.util.Map;
import java.util.Set;

/**
 * Abstract Log Processor that stores offsets for paritions and checks if a given event is already processed.
 */
public abstract class AbstractKafkaLogProcessor implements KafkaLogProcessor {

  private final Map<Integer, Checkpoint> partitonCheckpoints;
  public AbstractKafkaLogProcessor() {
    this.partitonCheckpoints = Maps.newHashMap();
  }

  public void init(Set<Integer> partitions, CheckpointManager checkpointManager) {
    partitonCheckpoints.clear();
    try {
      Map<Integer, Checkpoint> partitionMap = checkpointManager.getCheckpoint(partitions);
      for (Map.Entry<Integer, Checkpoint> partition : partitionMap.entrySet()) {
        partitonCheckpoints.put(partition.getKey(), partition.getValue());
      }
    } catch (Exception e) {
      throw Throwables.propagate(e);
    }
  }

  public void process(KafkaLogEvent event) {
    if (!alreadyProcessed(event)) {
      doProcess(event);
    }
  }

  /**
   * doProcess method will be called if the event is not already processed.
   *
   * @param event KafkaLogEvent
   */
  protected abstract void doProcess(KafkaLogEvent event);

  public boolean alreadyProcessed(KafkaLogEvent event) {
    // If no checkpoint is found, then the event needs to be processed.
    // if the event offset is less than or equal to what is already checkpointed then the event is already processed
    Checkpoint checkpoint = partitonCheckpoints.get(event.getPartition());
    return checkpoint != null && event.getNextOffset() <= checkpoint.getNextOffset();
  }
}