CommonStatsRolling.java

package org.act.tstream.stats;

import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

import backtype.storm.generated.GlobalStreamId;

import org.act.tstream.cluster.Common;
import org.act.tstream.common.stats.StatBuckets;
import org.act.tstream.common.stats.StaticsType;
import org.act.tstream.stats.rolling.RollingWindowSet;
import org.act.tstream.utils.EventSampler;
import org.act.tstream.utils.Pair;
import org.act.tstream.utils.TimeUtils;

/**
 * both spout and bolt will use base statics
 * 
 * @author Longda/yannian
 * 
 */
public class CommonStatsRolling {

	private static final long serialVersionUID = -2145444660360278001L;

	protected Map<StaticsType, RollingWindowSet> staticTypeMap = new HashMap<StaticsType, RollingWindowSet>();

	// <StremId, EventSampler>
	protected Map<String, EventSampler> emittedSamplers;
	protected Map<String, EventSampler> sendTpsSamplers;

	// <componentId, <streamId, EventSampler>>
	protected Map<String, Map<String, EventSampler>> recvTpsSamplers;

	protected Map<String, Map<String, EventSampler>> processSamplers;

	// in order to improve performance, use two type of samplers
	// protected Map<String, Map<String, EventSampler>> ackedSamplers;
	// protected Map<String, Map<String, EventSampler>> failedSamplers;
	protected Map<String, Map<String, EventSampler>> boltAckedSamplers;
	protected Map<String, Map<String, EventSampler>> boltFailedSamplers;
	protected Map<String, EventSampler> spoutAckedSamplers;
	protected Map<String, EventSampler> spoutFailedSamplers;

	protected boolean enable = true;

	protected Integer rate;

	public CommonStatsRolling(Integer rate) {
		RollingWindowSet emitted = StatFunction
				.keyed_counter_rolling_window_set(
						StatFunction.NUM_STAT_BUCKETS, StatBuckets.STAT_BUCKETS);
		staticTypeMap.put(StaticsType.emitted, emitted);

		RollingWindowSet sendTps = StatFunction.keyed_avg_rolling_window_set(
				StatFunction.NUM_STAT_BUCKETS, StatBuckets.STAT_BUCKETS);
		staticTypeMap.put(StaticsType.send_tps, sendTps);

		RollingWindowSet recvTps = StatFunction.keyed_avg_rolling_window_set(
				StatFunction.NUM_STAT_BUCKETS, StatBuckets.STAT_BUCKETS);
		staticTypeMap.put(StaticsType.recv_tps, recvTps);

		RollingWindowSet acked = StatFunction.keyed_counter_rolling_window_set(
				StatFunction.NUM_STAT_BUCKETS, StatBuckets.STAT_BUCKETS);
		staticTypeMap.put(StaticsType.acked, acked);

		RollingWindowSet failed = StatFunction
				.keyed_counter_rolling_window_set(
						StatFunction.NUM_STAT_BUCKETS, StatBuckets.STAT_BUCKETS);
		staticTypeMap.put(StaticsType.failed, failed);

		RollingWindowSet process_latencies = StatFunction
				.keyed_avg_rolling_window_set(StatFunction.NUM_STAT_BUCKETS,
						StatBuckets.STAT_BUCKETS);

		staticTypeMap.put(StaticsType.process_latencies, process_latencies);

		emittedSamplers = new HashMap<String, EventSampler>();
		sendTpsSamplers = new HashMap<String, EventSampler>();
		recvTpsSamplers = new HashMap<String, Map<String, EventSampler>>();
		boltAckedSamplers = new HashMap<String, Map<String, EventSampler>>();
		boltFailedSamplers = new HashMap<String, Map<String, EventSampler>>();
		spoutAckedSamplers = new HashMap<String, EventSampler>();
		spoutFailedSamplers = new HashMap<String, EventSampler>();
		processSamplers = new HashMap<String, Map<String, EventSampler>>();

		this.rate = rate;
	}

	/**
	 * update statics
	 * 
	 * @param common
	 * @param path
	 * @param args
	 */
	public void update_task_stat(StaticsType type, Object... args) {

		RollingWindowSet statics = staticTypeMap.get(type);
		if (statics != null) {
			statics.update_rolling_window_set(args);
		}
	}

	public void send_tuple(String stream, int num_out_tasks) {
		if (enable == false) {
			return;
		}

		if (num_out_tasks <= 0) {
			return;
		}

		EventSampler emittedSampler = emittedSamplers.get(stream);
		if (emittedSampler == null) {
			emittedSampler = new EventSampler(rate);
			emittedSamplers.put(stream, emittedSampler);
		}

		Integer times = emittedSampler.timesCheck();
		if (times != null) {
			update_task_stat(StaticsType.emitted, stream, times * num_out_tasks);
		}

		EventSampler sendTpsSampler = sendTpsSamplers.get(stream);
		if (sendTpsSampler == null) {
			sendTpsSampler = new EventSampler(rate);
			sendTpsSamplers.put(stream, sendTpsSampler);
		}
		Integer send = sendTpsSampler.tpsCheck();

		if (send != null) {

			update_task_stat(StaticsType.send_tps, stream, send * num_out_tasks);

		}
	}

	public void recv_tuple(String component, String stream) {
		if (enable == false) {
			return;
		}

		Map<String, EventSampler> componentSamplers = recvTpsSamplers
				.get(component);
		if (componentSamplers == null) {
			componentSamplers = new HashMap<String, EventSampler>();
			recvTpsSamplers.put(component, componentSamplers);
		}

		EventSampler sampler = componentSamplers.get(stream);
		if (sampler == null) {
			sampler = new EventSampler(rate);
			componentSamplers.put(stream, sampler);
		}

		Integer recv = sampler.tpsCheck();

		if (recv != null) {
			GlobalStreamId key = new GlobalStreamId(component, stream);
			update_task_stat(StaticsType.recv_tps, key, recv);
		}
	}

	public void bolt_acked_tuple(String component, String stream,
			Long latency_ms) {

		if (enable == false) {
			return;
		}

		if (latency_ms == null) {
			return;
		}

		Map<String, EventSampler> componentSamplers = boltAckedSamplers
				.get(component);
		if (componentSamplers == null) {
			componentSamplers = new HashMap<String, EventSampler>();
			boltAckedSamplers.put(component, componentSamplers);
		}

		EventSampler sampler = componentSamplers.get(stream);
		if (sampler == null) {
			sampler = new EventSampler(rate);
			componentSamplers.put(stream, sampler);
		}

		Pair<Integer, Double> pair = sampler.avgCheck(latency_ms * 1000);
		if (pair == null) {
			return;
		}
		
		long avgLatency = (long)((double)pair.getSecond());
		GlobalStreamId key = new GlobalStreamId(component, stream);
		update_task_stat(StaticsType.acked, key, pair.getFirst());
		update_task_stat(StaticsType.process_latencies, key, avgLatency);
	}

	public void bolt_failed_tuple(String component, String stream) {
		if (enable == false) {
			return;
		}

		Map<String, EventSampler> componentSamplers = boltFailedSamplers
				.get(component);
		if (componentSamplers == null) {
			componentSamplers = new HashMap<String, EventSampler>();
			boltFailedSamplers.put(component, componentSamplers);
		}

		EventSampler sampler = componentSamplers.get(stream);
		if (sampler == null) {
			sampler = new EventSampler(rate);
			componentSamplers.put(stream, sampler);
		}

		Integer times = sampler.timesCheck();
		if (times == null) {
			return;
		}

		GlobalStreamId key = new GlobalStreamId(component, stream);
		update_task_stat(StaticsType.failed, key, times);
	}

	public void spout_acked_tuple(String stream, long st) {

		if (enable == false) {
			return;
		}

		if (st == 0) {
			return;
		}

		EventSampler sampler = spoutAckedSamplers.get(stream);
		if (sampler == null) {
			sampler = new EventSampler(rate);
			spoutAckedSamplers.put(stream, sampler);
		}

		

		long latency_ms = TimeUtils.time_delta_ms(st);
		Pair<Integer, Double> pair = sampler.avgCheck(latency_ms * 1000);
		if (pair == null) {
			return;
		}
		
		long avgLatency = (long)((double)pair.getSecond());
		GlobalStreamId key = new GlobalStreamId(Common.ACKER_COMPONENT_ID,
				stream);
		update_task_stat(StaticsType.acked, key, pair.getFirst());
		update_task_stat(StaticsType.process_latencies, key, avgLatency);
	}

	public void spout_failed_tuple(String stream) {
		if (enable == false) {
			return;
		}

		EventSampler sampler = spoutFailedSamplers.get(stream);
		if (sampler == null) {
			sampler = new EventSampler(rate);
			spoutFailedSamplers.put(stream, sampler);
		}

		Integer times = sampler.timesCheck();
		if (times == null) {
			return;
		}

		GlobalStreamId key = new GlobalStreamId(Common.ACKER_COMPONENT_ID,
				stream);
		update_task_stat(StaticsType.failed, key, times);
	}

	public CommonStatsData render_stats() {

		cleanup_stats(false);

		CommonStatsData ret = new CommonStatsData();

		for (Entry<StaticsType, RollingWindowSet> entry : staticTypeMap
				.entrySet()) {
			StaticsType type = entry.getKey();
			RollingWindowSet rws = entry.getValue();

			Map<Integer, Object> csData = rws.value_rolling_window_set();

			ret.put(type, csData);
		}

		return ret;
	}

	protected void cleanup_common_stats() {

		for (Entry<StaticsType, RollingWindowSet> entry : staticTypeMap
				.entrySet()) {
			RollingWindowSet rws = entry.getValue();
			rws.cleanup_rolling_window_set();
		}

	}

	public void cleanup_stats(boolean skipcommon) {
		if (skipcommon == false) {
			cleanup_common_stats();
		}

	}

	public RollingWindowSet get(StaticsType type) {
		return staticTypeMap.get(type);
	}

	public void put(StaticsType type, RollingWindowSet statics) {
		staticTypeMap.put(type, statics);
	}

	public Integer getRate() {
		return rate;
	}

	public void setRate(Integer rate) {
		this.rate = rate;
	}

}