JsonContentAggregator.java

/**
 * Copyright 2015 Otto (GmbH & Co KG)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.ottogroup.bi.spqr.operator.json.aggregator;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.ottogroup.bi.spqr.exception.ComponentInitializationFailedException;
import com.ottogroup.bi.spqr.exception.RequiredInputMissingException;
import com.ottogroup.bi.spqr.operator.json.JsonContentType;
import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponentType;
import com.ottogroup.bi.spqr.pipeline.component.annotation.SPQRComponent;
import com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperator;
import com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy;
import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage;

/**
 * Aggregates content of JSON documents provided
 * @author mnxfst
 * @since Mar 17, 2015
 */
@SPQRComponent(type=MicroPipelineComponentType.DELAYED_RESPONSE_OPERATOR, name="jsonContentAggregator", version="0.0.1", description="Aggregates arbitrary JSON content")
public class JsonContentAggregator implements DelayedResponseOperator {

	/** our faithful logging facility .... ;-) */
	private static final Logger logger = Logger.getLogger(JsonContentAggregator.class);
	
	////////////////////////////////////////////////////////////////////////
	// available settings
	/** if provided the pipeline identifier is added to output document */
	public static final String CFG_PIPELINE_ID = "pipelineId";
	/** type assigned to each output document */
	public static final String CFG_DOCUMENT_TYPE = "documentType";
	/** store and forward raw data - default: true */
	public static final String CFG_FORWARD_RAW_DATA = "forwardRawData";
	/** prefix to all field settings - required: field.1.name, field.1.path and field.1.type (settings must use continuous enumeration starting with value 1 */
	public static final String CFG_FIELD_PREFIX = "field.";
	//
	////////////////////////////////////////////////////////////////////////

	/** component identifier assigned by caller */
	private String id = null;
	/** maps inbound strings into object representations and json strings vice versa */
	private final ObjectMapper jsonMapper = new ObjectMapper();
	/** identifier as assigned to surrounding pipeline */
	private String pipelineId = null;
	/** document identifier added to each output message */
	private String documentType = null;
	/** overall number of messages processed */ 
	private long messageCount = 0;
	/** number of messages processed since last result collection */	
	private long messagesSinceLastResult = 0;
	/** store and forward raw data - default: true */
	private boolean storeForwardRawData = true;
	/** fields considered to be relevant mapped to aggregator that must be applied to values - none = data is added to raw output only */
	private List<JsonContentAggregatorFieldSetting> fields = new ArrayList<>();
	/** result document - reset after specified duration */
	private JsonContentAggregatorResult resultDocument = null;

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#initialize(java.util.Properties)
	 */
	public void initialize(Properties properties) throws RequiredInputMissingException,	ComponentInitializationFailedException {
		
		/////////////////////////////////////////////////////////////////////////////////////
		// assign and validate properties
		if(StringUtils.isBlank(this.id))
			throw new RequiredInputMissingException("Missing required component identifier");
		
		this.pipelineId = StringUtils.trim(properties.getProperty(CFG_PIPELINE_ID));		
		this.documentType = StringUtils.trim(properties.getProperty(CFG_DOCUMENT_TYPE));
		if(StringUtils.equalsIgnoreCase(properties.getProperty(CFG_FORWARD_RAW_DATA), "false"))
			this.storeForwardRawData = false;
		

		for(int i = 1; i < Integer.MAX_VALUE; i++) {
			String name = properties.getProperty(CFG_FIELD_PREFIX + i + ".name");
			if(StringUtils.isBlank(name))
				break;
			
			String path = properties.getProperty(CFG_FIELD_PREFIX + i + ".path");
			String valueType = properties.getProperty(CFG_FIELD_PREFIX + i + ".type");
			
			this.fields.add(new JsonContentAggregatorFieldSetting(name, path.split("\\."), StringUtils.equalsIgnoreCase("STRING", valueType) ? JsonContentType.STRING : JsonContentType.NUMERICAL));
		}
		/////////////////////////////////////////////////////////////////////////////////////
		
		if(logger.isDebugEnabled())
			logger.debug("json content aggregator [id="+id+"] initialized");
		
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperator#onMessage(com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage)
	 */
	public void onMessage(StreamingDataMessage message) {		
		this.messageCount++;
		this.messagesSinceLastResult++;
		
		
		// do nothing if either the event or the body is empty
		if(message == null || message.getBody() == null || message.getBody().length < 1)
			return;
		
		JsonNode jsonNode = null;
		try {
			jsonNode = jsonMapper.readTree(message.getBody());
		} catch(IOException e) {
			logger.error("Failed to read message body to json node. Ignoring message. Error: " + e.getMessage());
		}
		
		// return null in case the message could not be parsed into 
		// an object representation - the underlying processor does
		// not forward any NULL messages
		if(jsonNode == null)
			return;
		
		// initialize the result document if not already done
		if(this.resultDocument == null)
			this.resultDocument = new JsonContentAggregatorResult(this.pipelineId, this.documentType);
		
		Map<String, Object> rawData = new HashMap<>();
		// step through fields considered to be relevant, extract values and apply aggregation function
		for(final JsonContentAggregatorFieldSetting fieldSettings : fields) {
			
			// switch between string and numerical field values
			// string values may be counted only
			// numerical field values must be summed, min and max computed AND counted 
			
			// string values may be counted only
			if(fieldSettings.getValueType() == JsonContentType.STRING) {

				try {
					// read value into string representation and add it to raw data dump
					String value = getTextFieldValue(jsonNode, fieldSettings.getPath());
					if(storeForwardRawData)
						rawData.put(fieldSettings.getField(), value);
					
					// count occurrences of value
					try {
						this.resultDocument.incAggregatedValue(fieldSettings.getField(), value, 1);
					} catch (RequiredInputMissingException e) {
						logger.error("Field '"+fieldSettings.getField()+"' not found in event. Ignoring value. Error: " +e.getMessage());
					}
				} catch(Exception e) {
				}
			} else if(fieldSettings.getValueType() == JsonContentType.NUMERICAL) {			
				
				try {
					// read value into numerical representation and add it to raw data map
					long value = getNumericalFieldValue(jsonNode, fieldSettings.getPath());
					if(storeForwardRawData)
						rawData.put(fieldSettings.getField(), value);
					
					// compute min, max and sum and add these values to result document
					try {
						this.resultDocument.evalMinAggregatedValue(fieldSettings.getField(), "min", value);
						this.resultDocument.evalMaxAggregatedValue(fieldSettings.getField(), "max", value);
						this.resultDocument.incAggregatedValue(fieldSettings.getField(), "sum", value);
					} catch(RequiredInputMissingException e) {
						logger.error("Field '"+fieldSettings.getField()+"' not found in event. Ignoring value. Error: " +e.getMessage());
					}				
					
				} catch(Exception e) {
				}
			}			
		}
		
		// add raw data to document
		if(storeForwardRawData)
			this.resultDocument.addRawData(rawData);
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperator#getResult()
	 */
	public StreamingDataMessage[] getResult() {
		this.messagesSinceLastResult = 0;
		
		StreamingDataMessage message = null;
		try {			
			message = new StreamingDataMessage(jsonMapper.writeValueAsBytes(this.resultDocument), System.currentTimeMillis());
		} catch (JsonProcessingException e) {
			logger.error("Failed to convert result document into JSON");
		}
		this.resultDocument = new JsonContentAggregatorResult(this.pipelineId, this.documentType);
		return new StreamingDataMessage[]{message};
	}

	/**
	 * Walks along the path provided and reads out the leaf value which is returned as string 
	 * @param jsonNode
	 * @param fieldPath
	 * @return
	 */
	protected String getTextFieldValue(final JsonNode jsonNode, final String[] fieldPath) {

		int fieldAccessStep = 0;
		JsonNode contentNode = jsonNode;
		while(fieldAccessStep < fieldPath.length) {
			contentNode = contentNode.get(fieldPath[fieldAccessStep]);
			fieldAccessStep++;
		}	

		return contentNode.textValue();
	}
	
	/**
	 * Walks along the path provided and reads out the leaf value which is returned as long value
	 * @param jsonNode
	 * @param fieldPath
	 * @return
	 */
	protected long getNumericalFieldValue(final JsonNode jsonNode, final String[] fieldPath) {

		int fieldAccessStep = 0;
		JsonNode contentNode = jsonNode;
		while(fieldAccessStep < fieldPath.length) {
			contentNode = contentNode.get(fieldPath[fieldAccessStep]);
			fieldAccessStep++;
		}	

		return contentNode.asLong();
	}
	
	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperator#setWaitStrategy(com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy)
	 */
	public void setWaitStrategy(DelayedResponseOperatorWaitStrategy waitStrategy) {
		// do nothing as the reference is not required
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#shutdown()
	 */
	public boolean shutdown() {
		return true;
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperator#getNumberOfMessagesSinceLastResult()
	 */
	public long getNumberOfMessagesSinceLastResult() {
		return this.messagesSinceLastResult;
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#getType()
	 */
	public MicroPipelineComponentType getType() {
		return MicroPipelineComponentType.DELAYED_RESPONSE_OPERATOR;
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.operator.Operator#getTotalNumOfMessages()
	 */
	public long getTotalNumOfMessages() {
		return this.messageCount;
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#setId(java.lang.String)
	 */
	public void setId(String id) {
		this.id = id;
	}

	/**
	 * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#getId()
	 */
	public String getId() {
		return this.id;
	}

}