ExternalFeedToKafkaAdapterSpout.java

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import java.util.Map;
import java.util.Properties;

/**
 * A
 */
public class ExternalFeedToKafkaAdapterSpout extends BaseRichSpout {
    private static final long serialVersionUID = 1L;
    public static final String RECORD = "record";


    private String brokerConnectString;

    private String topicName;
    private String serializerClass;

    private transient SpoutOutputCollector collector;
    private transient TopologyContext context;
    private transient Producer<String, String> producer;

    private IFeedItemProvider feedProvider;

    public ExternalFeedToKafkaAdapterSpout(IFeedItemProvider feedProvider,
                                           String brokerConnectString,
                                           String topicName,
                                           String serializerClass) {
        this.feedProvider = feedProvider;
        this.brokerConnectString = brokerConnectString;
        this.topicName = topicName;
        if (serializerClass == null) {
            serializerClass = "kafka.serializer.StringEncoder";
        }
        this.serializerClass = serializerClass;
    }


    public void setFeedProvider(IFeedItemProvider feedProvider) { // mainly for testing
        this.feedProvider = feedProvider;
    }

    public void setTopicName(String topicName) { // mainly for testing
        this.topicName = topicName;
    }


    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields(RECORD));
    }


    @Override
    public void open(@SuppressWarnings("rawtypes") Map conf,
                     TopologyContext context,
                     SpoutOutputCollector collector) {
        this.collector = collector;
        this.context = context;

        producer = setupProducer();

        Thread feedProviderThread =
                new Thread(feedProvider.getRunnableTask(), "feedProviderThread");
        feedProviderThread.start();
    }


    @Override
    public void nextTuple() {
        try {
                Thread.sleep(10);
            } catch (InterruptedException e) {
                e.printStackTrace();
        }

        Object feedItem = feedProvider.getNextItemIfAvailable();

        if (feedItem != null) {
            System.out.println(">>>->>feed item is: " + feedItem);
            final Map<String, Object> itemAsMap = ImmutableMap.of(RECORD, feedItem);
            try {
                String itemAsJson = new Gson().toJson(itemAsMap);
                KeyedMessage<String, String> data =
                        new KeyedMessage<String, String>(topicName, itemAsJson);
                producer.send(data);
            } catch (Exception e) {
                throw new RuntimeException("Conversion to json failed: " + feedItem);
            }

        } else {
            try {
                Thread.sleep(10);
            } catch (InterruptedException e) {
                e.printStackTrace();   // do something more meaningful here?
            }
        }

    }

    // should be private, but have not gotten PowerMock unit testing to work yet.
    protected Producer<String, String> setupProducer() {
        Properties props = new Properties();
        props.put("metadata.broker.list", brokerConnectString);
        props.put("serializer.class", serializerClass);
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        props.put("producer.type", "sync");
        props.put("batch.size", "1");

        ProducerConfig config = new ProducerConfig(props);
        return new Producer<String, String>(config);
    }
}