package com.alipay.bluewhale.core.task.executer; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.log4j.Logger; import backtype.storm.Config; import backtype.storm.serialization.KryoTupleDeserializer; import backtype.storm.spout.ISpoutOutputCollector; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Tuple; import backtype.storm.utils.Time; import backtype.storm.utils.TimeCacheMap; import com.alipay.bluewhale.core.callback.RunnableCallback; import com.alipay.bluewhale.core.messaging.IConnection; import com.alipay.bluewhale.core.stats.BaseTaskStatsRolling; import com.alipay.bluewhale.core.task.acker.Acker; import com.alipay.bluewhale.core.task.transfer.TaskSendTargets; import com.alipay.bluewhale.core.task.transfer.TupleInfo; import com.alipay.bluewhale.core.utils.StormUtils; import com.alipay.bluewhale.core.utils.TimeUtils; import com.alipay.bluewhale.core.work.transfer.WorkerTransfer; /** * spout�����������ϵĵ���spout��nextTuple���� * ִ��acker�Ĵ�������������spout��ack��fail���� * @author yannian * */ public class SpoutExecutors extends RunnableCallback { private static Logger LOG = Logger.getLogger(SpoutExecutors.class); private Map storm_conf; private IConnection puller; private AtomicBoolean zkActive; private TopologyContext user_context; private BaseTaskStatsRolling task_stats; private ConcurrentLinkedQueue<Runnable> event_queue; private backtype.storm.spout.ISpout spout; private TimeCacheMap pending; private ISpoutOutputCollector output_collector; private Integer max_spout_pending; private KryoTupleDeserializer deserializer; private String component_id; private AtomicBoolean isRecvRun; private Object lockrecv=new Object(); private Exception error=null; public SpoutExecutors(backtype.storm.spout.ISpout _spout, WorkerTransfer _transfer_fn, Map _storm_conf, IConnection _puller, TaskSendTargets sendTargets, AtomicBoolean _storm_active_atom, TopologyContext topology_context, TopologyContext _user_context, BaseTaskStatsRolling _task_stats) { this.spout = _spout; this.storm_conf = _storm_conf; this.puller = _puller; this.zkActive = _storm_active_atom; this.user_context = _user_context; this.task_stats = _task_stats; Integer task_id = topology_context.getThisTaskId(); this.component_id = topology_context.getThisComponentId(); this.max_spout_pending = StormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING)); this.deserializer = new KryoTupleDeserializer(storm_conf,topology_context);// (KryoTupleDeserializer. // storm-conf this.event_queue = new ConcurrentLinkedQueue<Runnable>(); int message_timeout_secs =StormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); // �������ش���tuple�Ķ��У�����ȴ������tuple̫������spout����ͣ��TimeCacheMap�����tuple����ʱ�������tuple.fail��������֪ͨacker�ط���typle this.pending = new TimeCacheMap(message_timeout_secs,new SpoutTimeCallBack<Object, Object>(event_queue, spout,storm_conf,task_stats)); // ����collector,ʵ�����ǵ���send_spout_msg this.output_collector = new SpoutCollector(task_id,spout,task_stats,sendTargets,storm_conf,_transfer_fn,pending,topology_context,event_queue); LOG.info("Opening spout " + component_id + ":" + task_id); this.spout.open(storm_conf, user_context, new SpoutOutputCollector(output_collector)); LOG.info("Opend spout " + component_id + ":" + task_id); this.isRecvRun=new AtomicBoolean(); } private boolean IsActive() { return zkActive.get(); } private void executeEvent() { while (true) { // ��event_querey��ĵ���acker��ack��fail��Ϣ���͹�ȥ Runnable event = event_queue.poll(); if (event != null) { event.run(); } else { break; } } } private void executeNextTupe() { // ������г��ȣ�С���趨�Ļ�������С����ôִ��spout.nextTuple(); if (max_spout_pending == null || pending.size() < max_spout_pending) { if (this.IsActive()) { try { spout.nextTuple(); } catch (RuntimeException e) { error = e; LOG.error("spout execute error ", e); } catch (Exception e) { error = e; LOG.error("spout execute error ", e); } } else { try { Time.sleep(100); } catch (InterruptedException e) { LOG.error("spout sleep error ", e); } } } } private void recv() { // �ӱ���zeroMq����Ӧ����Ϣ���ֱ���event_queue�����ack��fail��Ϣ while (true) { byte[] ser_msg = puller.recv(); if (ser_msg != null && ser_msg.length > 0) { Tuple tuple = deserializer.deserialize(ser_msg); Object id = tuple.getValue(0); Object olist = pending.remove(id); if (olist == null) { continue; } List list = (List) olist; Object start_time_ms = list.get(2); Long time_delta = null; if (start_time_ms != null) { time_delta = TimeUtils.time_delta_ms((Long) start_time_ms); } Object msgId = list.get(0); Object tupleInfo = list.get(1); if (msgId == null||tupleInfo==null) { continue; } String stream_id = tuple.getSourceStreamId(); if (stream_id.equals(Acker.ACKER_ACK_STREAM_ID)) { event_queue.add(new AckSpoutMsg(spout, storm_conf, msgId, (TupleInfo)tupleInfo, time_delta, task_stats)); } else if (stream_id.equals(Acker.ACKER_FAIL_STREAM_ID)) { event_queue.add(new FailSpoutMsg(spout, storm_conf, msgId, (TupleInfo)tupleInfo, time_delta, task_stats)); } } } } @Override public void run() { this.executeEvent(); this.executeNextTupe(); synchronized (lockrecv) { if (!SpoutExecutors.this.isRecvRun.get()) { SpoutExecutors.this.isRecvRun.set(true); new Thread(new Runnable() { @Override public void run() { try { SpoutExecutors.this.recv(); } finally { SpoutExecutors.this.isRecvRun.set(false); } } }).start(); } } } @Override public Object getResult() { if (this.IsActive()) { return 0; } else { return -1; } } @Override public Exception error() { return error; } }