KafkaWriter.java

/**
 * The MIT License
 * Copyright © 2010 JmxTrans team
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package com.googlecode.jmxtrans.model.output.kafka;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.googlecode.jmxtrans.model.Query;
import com.googlecode.jmxtrans.model.Result;
import com.googlecode.jmxtrans.model.Server;
import com.googlecode.jmxtrans.model.ValidationException;
import com.googlecode.jmxtrans.model.output.BaseOutputWriter;
import com.googlecode.jmxtrans.model.output.Settings;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.concurrent.NotThreadSafe;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;

import static com.fasterxml.jackson.core.JsonEncoding.UTF8;
import static com.googlecode.jmxtrans.model.naming.KeyUtils.getKeyString;
import static com.googlecode.jmxtrans.util.NumberUtils.isNumeric;
import static java.util.Arrays.asList;

/**
 * This low latency and thread safe output writer sends data to a kafka topics in JSON format.
 * Kafka Topics can be passed as separated by commas such as kafka01,kafka02.
 *
 * @author : utkarsh bhatnagar
 * @github user : utkarshcmu
 * @email : utkarsh.cmu@gmail.com
 */

@NotThreadSafe
public class KafkaWriter extends BaseOutputWriter {

	private static final Logger log = LoggerFactory.getLogger(KafkaWriter.class);

	private static final String DEFAULT_ROOT_PREFIX = "servers";
	private final JsonFactory jsonFactory;

	private Producer<String,String> producer;
	private final Iterable<String> topics;
	private final String rootPrefix;
	private final ImmutableMap<String, String> tags;

	@JsonCreator
	public KafkaWriter(
			@JsonProperty("typeNames") ImmutableList<String> typeNames,
			@JsonProperty("booleanAsNumber") boolean booleanAsNumber,
			@JsonProperty("rootPrefix") String rootPrefix,
			@JsonProperty("debug") Boolean debugEnabled,
			@JsonProperty("topics") String topics,
			@JsonProperty("tags") Map<String, String> tags,
			@JsonProperty("settings") Map<String, Object> settings) {
		super(typeNames, booleanAsNumber, debugEnabled, settings);
		this.rootPrefix = firstNonNull(
						rootPrefix,
						(String) getSettings().get("rootPrefix"),
						DEFAULT_ROOT_PREFIX);
		// Setting all the required Kafka Properties
		Properties kafkaProperties =  new Properties();
		kafkaProperties.setProperty("metadata.broker.list", Settings.getStringSetting(settings, "metadata.broker.list", null));
		kafkaProperties.setProperty("zk.connect", Settings.getStringSetting(settings, "zk.connect", null));
		kafkaProperties.setProperty("serializer.class", Settings.getStringSetting(settings, "serializer.class", null));
		this.producer= new Producer<>(new ProducerConfig(kafkaProperties));
		this.topics = asList(Settings.getStringSetting(settings, "topics", "").split(","));
		this.tags = ImmutableMap.copyOf(firstNonNull(tags, (Map<String, String>) getSettings().get("tags"), ImmutableMap.<String, String>of()));
		jsonFactory = new JsonFactory();
	}

	@Override
	public void validateSetup(Server server, Query query) throws ValidationException {}

	@Override
	protected void internalWrite(Server server, Query query, ImmutableList<Result> results) throws Exception {
		List<String> typeNames = this.getTypeNames();

		for (Result result : results) {
			log.debug("Query result: [{}]", result);
			Map<String, Object> resultValues = result.getValues();
			for (Entry<String, Object> values : resultValues.entrySet()) {
				Object value = values.getValue();
				if (isNumeric(value)) {
					String message = createJsonMessage(server, query, typeNames, result, values, value);
					for(String topic : this.topics) {
						log.debug("Topic: [{}] ; Kafka Message: [{}]", topic, message);
						producer.send(new KeyedMessage<String, String>(topic, message));
					}
				} else {
					log.warn("Unable to submit non-numeric value to Kafka: [{}] from result [{}]", value, result);
				}
			}
		}
	}

	private String createJsonMessage(Server server, Query query, List<String> typeNames, Result result, Entry<String, Object> values, Object value) throws IOException {
		String keyString = getKeyString(server, query, result, values, typeNames, this.rootPrefix);
		String cleanKeyString = keyString.replaceAll("[()]", "_");

		try (
			ByteArrayOutputStream out = new ByteArrayOutputStream();
			JsonGenerator generator = jsonFactory.createGenerator(out, UTF8)
		){
			generator.writeStartObject();
			generator.writeStringField("keyspace", cleanKeyString);
			generator.writeStringField("value", value.toString());
			generator.writeNumberField("timestamp", result.getEpoch() / 1000);
			generator.writeObjectFieldStart("tags");

			for (Entry<String, String> tag : this.tags.entrySet()) {
				generator.writeStringField(tag.getKey(), tag.getValue());
			}

			generator.writeEndObject();
			generator.writeEndObject();
			generator.close();
			return out.toString("UTF-8");
		}
	}

	@VisibleForTesting
	void setProducer(Producer<String, String> producer) {
		this.producer = producer;
	}

}