/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package backtype.storm.transactional.partitioned; import backtype.storm.Config; import backtype.storm.coordination.BatchOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.transactional.ICommitterTransactionalSpout; import backtype.storm.transactional.ITransactionalSpout; import backtype.storm.transactional.TransactionAttempt; import backtype.storm.transactional.state.RotatingTransactionalState; import backtype.storm.transactional.state.TransactionalState; import java.math.BigInteger; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; public class OpaquePartitionedTransactionalSpoutExecutor implements ICommitterTransactionalSpout<Object> { IOpaquePartitionedTransactionalSpout _spout; public class Coordinator implements ITransactionalSpout.Coordinator<Object> { IOpaquePartitionedTransactionalSpout.Coordinator _coordinator; public Coordinator(Map conf, TopologyContext context) { _coordinator = _spout.getCoordinator(conf, context); } @Override public Object initializeTransaction(BigInteger txid, Object prevMetadata) { return null; } @Override public boolean isReady() { return _coordinator.isReady(); } @Override public void close() { _coordinator.close(); } } public class Emitter implements ICommitterTransactionalSpout.Emitter { IOpaquePartitionedTransactionalSpout.Emitter _emitter; TransactionalState _state; TreeMap<BigInteger, Map<Integer, Object>> _cachedMetas = new TreeMap<BigInteger, Map<Integer, Object>>(); Map<Integer, RotatingTransactionalState> _partitionStates = new HashMap<Integer, RotatingTransactionalState>(); int _index; int _numTasks; public Emitter(Map conf, TopologyContext context) { _emitter = _spout.getEmitter(conf, context); _index = context.getThisTaskIndex(); _numTasks = context.getComponentTasks(context.getThisComponentId()).size(); _state = TransactionalState.newUserState(conf, (String) conf.get(Config.TOPOLOGY_TRANSACTIONAL_ID), getComponentConfiguration()); List<String> existingPartitions = _state.list(""); for (String p : existingPartitions) { int partition = Integer.parseInt(p); if ((partition - _index) % _numTasks == 0) { _partitionStates.put(partition, new RotatingTransactionalState(_state, p)); } } } @Override public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, BatchOutputCollector collector) { Map<Integer, Object> metas = new HashMap<Integer, Object>(); _cachedMetas.put(tx.getTransactionId(), metas); int partitions = _emitter.numPartitions(); Entry<BigInteger, Map<Integer, Object>> entry = _cachedMetas.lowerEntry(tx.getTransactionId()); Map<Integer, Object> prevCached; if (entry != null) { prevCached = entry.getValue(); } else { prevCached = new HashMap<Integer, Object>(); } for (int i = _index; i < partitions; i += _numTasks) { RotatingTransactionalState state = _partitionStates.get(i); if (state == null) { state = new RotatingTransactionalState(_state, "" + i); _partitionStates.put(i, state); } state.removeState(tx.getTransactionId()); Object lastMeta = prevCached.get(i); if (lastMeta == null) lastMeta = state.getLastState(); Object meta = _emitter.emitPartitionBatch(tx, collector, i, lastMeta); metas.put(i, meta); } } @Override public void cleanupBefore(BigInteger txid) { for (RotatingTransactionalState state : _partitionStates.values()) { state.cleanupBefore(txid); } } @Override public void commit(TransactionAttempt attempt) { BigInteger txid = attempt.getTransactionId(); Map<Integer, Object> metas = _cachedMetas.remove(txid); for (Integer partition : metas.keySet()) { Object meta = metas.get(partition); _partitionStates.get(partition).overrideState(txid, meta); } } @Override public void close() { _emitter.close(); } } public OpaquePartitionedTransactionalSpoutExecutor(IOpaquePartitionedTransactionalSpout spout) { _spout = spout; } @Override public ITransactionalSpout.Coordinator<Object> getCoordinator(Map conf, TopologyContext context) { return new Coordinator(conf, context); } @Override public ICommitterTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) { return new Emitter(conf, context); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { _spout.declareOutputFields(declarer); } @Override public Map<String, Object> getComponentConfiguration() { return _spout.getComponentConfiguration(); } }