KafkaTSDataConsumer.java

package com.easemob.tsdb.kafka.plugin;

import com.easemob.tsdb.thrift.models.TSData;
import com.easemob.tsdb.thrift.rpc.service.TSDBDelegate;
import kafka.consumer.KafkaStream;
import kafka.message.MessageAndMetadata;
import org.apache.thrift.TDeserializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;

import static com.easemob.tsdb.utils.TSDataUtils.*;

/**
 * Description for Class KafkaTSDataTopicConsumer
 *
 * @author Lynch Lee<Lynch.lee9527@gmail.com>
 * @version 2016-01-14.
 */
public class KafkaTSDataConsumer extends AbstractKafkaConsumer {

    private static final Logger logger = LoggerFactory.getLogger(KafkaTSDataConsumer.class);

    /**
     * 按照设计, 每个topic一个线程, 线程内跑一个任务, 任务内的业务逻辑内容由KafkaPlaintextConsumer或者KafkaTSDataConsumer来指定.
     * 此计数器应该针对单个任务
     */
    private final AtomicLong tsdataMetricsCounter = new AtomicLong();
    private final TDeserializer deserializer = new TDeserializer();


    public KafkaTSDataConsumer(String tsdataTopic, TSDBDelegate tsdb, boolean skipTSDB,
                               Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap) {
        super(tsdb, tsdataTopic, skipTSDB, consumerMap);
    }


    public void persistMetrics(KafkaStream<byte[], byte[]> kafkaStreams) {
        for (MessageAndMetadata<byte[], byte[]> message : kafkaStreams) {

            TSData tsData = new TSData();

            try {
                deserializer.deserialize(tsData, message.message());

                if (logger.isDebugEnabled()) {
                    logger.debug("persisting TSData metrics : {}, queue size is {}", tsData);
                }

                if (skipTSDB) {
                    if (logger.isDebugEnabled()) {
                        logger.debug("skipping persist {} to OpenTSDB", tsData);
                    }
                } else {
                    if (isValidMetricAndTags(tsData.getName(), tsData.getTags())) {
                        return;
                    }

                    metricsService.putTSData(tsData);
                    tsdataMetricsCounter.incrementAndGet();
                }

                if (logger.isDebugEnabled()) {
                    logger.debug("persisting TSData metrics to queue done");
                }
            } catch (Exception e) {
                logger.error("Failed to put tsdata metrics {}", tsData, e);
            }
        }
    }

}