/*
* Galaxy
* Copyright (c) 2012-2014, Parallel Universe Software Co. All rights reserved.
*
* This program and the accompanying materials are dual-licensed under
* either the terms of the Eclipse Public License v1.0 as published by
* the Eclipse Foundation
*
* or (per the licensee's choosing)
*
* under the terms of the GNU Lesser General Public License version 3.0
* as published by the Free Software Foundation.
*/
package co.paralleluniverse.galaxy.netty;
import co.paralleluniverse.common.monitoring.ThreadPoolExecutorMonitor;
import co.paralleluniverse.galaxy.Cluster;
import co.paralleluniverse.galaxy.cluster.ReaderWriters;
import co.paralleluniverse.galaxy.core.AbstractComm;
import co.paralleluniverse.galaxy.core.Comm;
import co.paralleluniverse.galaxy.core.CommThread;
import co.paralleluniverse.galaxy.core.Message;
import co.paralleluniverse.galaxy.core.Message.LineMessage;
import co.paralleluniverse.galaxy.core.MessageReceiver;
import co.paralleluniverse.galaxy.core.NodeNotFoundException;
import co.paralleluniverse.galaxy.core.ServerComm;
import static co.paralleluniverse.galaxy.netty.IpConstants.*;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongSet;
import it.unimi.dsi.fastutil.shorts.ShortIterator;
import it.unimi.dsi.fastutil.shorts.ShortOpenHashSet;
import it.unimi.dsi.fastutil.shorts.ShortSet;
import java.beans.ConstructorProperties;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.NetworkInterface;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import static java.util.concurrent.TimeUnit.*;
import org.jboss.netty.bootstrap.ConnectionlessBootstrap;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.FixedReceiveBufferSizePredictorFactory;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.channel.socket.DatagramChannel;
import org.jboss.netty.channel.socket.DatagramChannelFactory;
import org.jboss.netty.channel.socket.nio.NioDatagramChannelFactory;
import org.jboss.netty.channel.socket.oio.OioDatagramChannelFactory;
import org.jboss.netty.handler.execution.OrderedMemoryAwareThreadPoolExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jmx.export.annotation.ManagedAttribute;
/**
* This crucial class could use a good refactoring.
*
* @author pron
*/
public class UDPComm extends AbstractComm<InetSocketAddress> {
// Note: class must be public for Spring's auto generated javax.management.modelmbean.RequiredModelMBean to expose @ManagedAttribute
private static final Logger LOG = LoggerFactory.getLogger(UDPComm.class);
//
private final int port;
private InetSocketAddress multicastGroup;
private NetworkInterface multicastNetworkInterface;
private int maxQueueSize = 50;
private int maxPacketSize = 4096;
private int maxRequestOnlyPacketSize = maxPacketSize / 2;
private long minDelayNanos = NANOSECONDS.convert(1, MILLISECONDS);
private long maxDelayNanos = NANOSECONDS.convert(10, MILLISECONDS);
private long resendPeriodNanos = NANOSECONDS.convert(20, MILLISECONDS);
private boolean jitter = false;
private boolean exponentialBackoff = true;
private int minimumNodesToMulticast = 3;
private ThreadPoolExecutor workerExecutor;
private OrderedMemoryAwareThreadPoolExecutor receiveExecutor;
//
private final Comm serverComm;
private DatagramChannelFactory channelFactory;
private ConnectionlessBootstrap bootstrap;
private DatagramChannel channel;
private DatagramChannel multicastChannel;
private BroadcastPeer broadcastPeer = new BroadcastPeer();
private SocketAddress myAddress;
private final ConcurrentMap<Short, NodePeer> peers = new ConcurrentHashMap<Short, NodePeer>();
private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("uspCommScheduled-%d").setDaemon(true).build());
private final UDPCommMonitor monitor;
@ConstructorProperties({"name", "cluster", "serverComm", "port"})
UDPComm(String name, Cluster cluster, ServerComm serverComm, int port) throws Exception {
super(name, cluster, new SocketNodeAddressResolver(cluster, IP_COMM_PORT));
this.serverComm = serverComm;
this.port = port;
cluster.addNodeProperty(IP_ADDRESS, true, true, INET_ADDRESS_READER_WRITER);
cluster.setNodeProperty(IP_ADDRESS, InetAddress.getLocalHost());
cluster.addNodeProperty(IP_COMM_PORT, true, false, ReaderWriters.INTEGER);
cluster.setNodeProperty(IP_COMM_PORT, port);
this.monitor = new UDPCommMonitor(name, this);
}
@ManagedAttribute
public int getPort() {
return port;
}
public void setReceiveBufferSize(int size) {
assertDuringInitialization();
bootstrap.setOption("receiveBufferSize", size);
}
public void setMulticastGroup(InetSocketAddress group) {
assertDuringInitialization();
this.multicastGroup = group;
}
@ManagedAttribute
public String getMulticastGroupName() {
return multicastGroup.toString();
}
public void setMulticastNetworkInterface(NetworkInterface multicastNetworkInterface) {
assertDuringInitialization();
this.multicastNetworkInterface = multicastNetworkInterface;
}
@ManagedAttribute
public String getMulticastNetworkInterfaceName() {
return multicastNetworkInterface.toString();
}
public void setMaxQueueSize(int maxQueueSize) {
assertDuringInitialization();
this.maxQueueSize = maxQueueSize;
}
@ManagedAttribute
public int getMaxQueueSize() {
return maxQueueSize;
}
public void setMaxPacketSize(int maxPacketSize) {
assertDuringInitialization();
this.maxPacketSize = maxPacketSize;
}
@ManagedAttribute
public int getMaxPacketSize() {
return maxPacketSize;
}
public void setMaxRequestOnlyPacketSize(int maxRequestOnlyPacketSize) {
assertDuringInitialization();
this.maxRequestOnlyPacketSize = maxRequestOnlyPacketSize;
}
@ManagedAttribute
public int getMaxRequestOnlyPacketSize() {
return maxRequestOnlyPacketSize;
}
public void setMaxDelayMicrosecs(int maxDelayMicrosecs) {
assertDuringInitialization();
this.maxDelayNanos = NANOSECONDS.convert(maxDelayMicrosecs, MICROSECONDS);
}
@ManagedAttribute
public int getMaxDelayMicrosecs() {
return (int) MICROSECONDS.convert(maxDelayNanos, NANOSECONDS);
}
public void setMinDelayMicrosecs(int minDelayMicrosecs) {
assertDuringInitialization();
this.minDelayNanos = NANOSECONDS.convert(minDelayMicrosecs, MICROSECONDS);
}
@ManagedAttribute
public int getMinDelayMicrosecs() {
return (int) MICROSECONDS.convert(minDelayNanos, NANOSECONDS);
}
public void setResendPeriodMillisecs(int resnedPeriodMillisecs) {
assertDuringInitialization();
this.resendPeriodNanos = NANOSECONDS.convert(resnedPeriodMillisecs, MILLISECONDS);
}
@ManagedAttribute
public int getResendPeriodMillisecs() {
return (int) MILLISECONDS.convert(resendPeriodNanos, NANOSECONDS);
}
public void setMinimumNodesToMulticast(int minimumNodesToMulticast) {
assertDuringInitialization();
this.minimumNodesToMulticast = minimumNodesToMulticast;
}
@ManagedAttribute
public int getMinimumNodesToMulticast() {
return minimumNodesToMulticast;
}
public void setWorkerExecutor(ThreadPoolExecutor executor) {
assertDuringInitialization();
this.workerExecutor = executor;
}
@ManagedAttribute
public String getWorkerExecutorName() {
return "udpCommWorkerExecutor";
}
public void setReceiveExecutor(OrderedMemoryAwareThreadPoolExecutor executor) {
assertDuringInitialization();
this.receiveExecutor = executor;
}
@ManagedAttribute
public String getReceiveExecutorName() {
return "udpCommReceiveExecutor";
}
public void setJitter(boolean value) {
// see http://highscalability.com/blog/2012/4/17/youtube-strategy-adding-jitter-isnt-a-bug.html and http://news.ycombinator.com/item?id=3757456
assertDuringInitialization();
this.jitter = value;
}
@ManagedAttribute
public boolean isJitter() {
return jitter;
}
public void setExponentialBackoff(boolean value) {
assertDuringInitialization();
this.exponentialBackoff = value;
}
@ManagedAttribute
public boolean isExponentialBackoff() {
return exponentialBackoff;
}
@Override
public void setReceiver(MessageReceiver receiver) {
super.setReceiver(receiver);
if (serverComm != null)
serverComm.setReceiver(receiver);
}
@Override
public void init() throws Exception {
super.init();
if (!isSendToServerInsteadOfMulticast() && multicastGroup == null) {
LOG.error("If sendToServerInsteadOfBroadcast, multicastGroup must be set!");
throw new RuntimeException("multicastGroup not set.");
}
this.myAddress = new InetSocketAddress(InetAddress.getLocalHost(), port);
if (workerExecutor == null)
workerExecutor = (ThreadPoolExecutor) Executors.newCachedThreadPool();
// Netty ignores executor thread naming strategy cause of Worker renaming policy.
// org.jboss.netty.channel.socket.nio.AbstractNioWorker.newThreadRenamingRunnable()
// And unfortunately for NioDatagramChannelFactory itsn't possible to pass our own ThreadNameDeterminer.
configureThreadPool(getWorkerExecutorName(), workerExecutor);
if (receiveExecutor != null)
configureThreadPool(getReceiveExecutorName(), receiveExecutor);
this.channelFactory = isSendToServerInsteadOfMulticast()
? new NioDatagramChannelFactory(workerExecutor, NettyUtils.getWorkerCount(workerExecutor))
: new OioDatagramChannelFactory(workerExecutor);
this.bootstrap = new ConnectionlessBootstrap(channelFactory);
this.bootstrap.setOption("receiveBufferSizePredictorFactory", new FixedReceiveBufferSizePredictorFactory(4096));
bootstrap.setPipelineFactory(new UdpMessagePipelineFactory(LOG, new ChannelNodeAddressResolver(addressResolver), receiveExecutor) {
@Override
public ChannelPipeline getPipeline() throws Exception {
final ChannelPipeline pipeline = super.getPipeline();
pipeline.addLast("router", new SimpleChannelHandler() {
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
if (ctx.getChannel() == multicastChannel) {
if (e.getRemoteAddress().equals(myAddress))
return; // this is our own multicast
((MessagePacket) e.getMessage()).setMulticast();
}
UDPComm.this.messageReceived((MessagePacket) e.getMessage());
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
LOG.info("Channel exception: {} {}", e.getCause().getClass().getName(), e.getCause().getMessage());
LOG.debug("Channel exception", e.getCause());
}
});
return pipeline;
}
});
bootstrap.setOption("localAddress", new InetSocketAddress(InetAddress.getLocalHost(), port));
bootstrap.setOption("tcpNoDelay", true);
monitor.registerMBean();
}
private void configureThreadPool(String name, ThreadPoolExecutor executor) {
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
executor.setThreadFactory(new ThreadFactoryBuilder().setNameFormat(name + "-%d").setDaemon(true).setThreadFactory(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
return new CommThread(r);
}
}).build());
ThreadPoolExecutorMonitor.register(name, executor);
}
@Override
public void postInit() throws Exception {
if (!sendToServerInsteadOfMulticast)
this.broadcastPeer = new BroadcastPeer();
super.postInit();
}
@Override
public void start(boolean master) {
this.channel = (DatagramChannel) bootstrap.bind();
LOG.info("Channel {} listening on port {}", channel, port);
if (!isSendToServerInsteadOfMulticast()) {
final int multicastPort = multicastGroup.getPort();
this.multicastChannel = (DatagramChannel) bootstrap.bind(new InetSocketAddress(multicastPort));
if (multicastNetworkInterface != null) {
LOG.info("Channel {} joining multicast group {} on network interface {}", new Object[]{multicastChannel, multicastGroup, multicastNetworkInterface});
multicastChannel.joinGroup(multicastGroup, multicastNetworkInterface);
} else {
LOG.info("Channel {} joining multicast group {} ", multicastChannel, multicastGroup);
multicastChannel.joinGroup(multicastGroup.getAddress());
}
} else
this.multicastChannel = null;
setReady(true);
}
@Override
public void shutdown() {
super.shutdown();
LOG.info("Shutting down.");
monitor.unregisterMBean();
if (channel != null)
channel.close();
if (multicastChannel != null)
multicastChannel.close();
channelFactory.releaseExternalResources();
}
// for testing only
void setChannel(DatagramChannel channel) {
this.channel = channel;
}
ExecutorService getExecutor() {
return executor;
}
@Override
protected void sendToServer(Message message) {
super.sendToServer(message);
try {
serverComm.send(message);
} catch (NodeNotFoundException e) {
throw new RuntimeException("Server not found!", e);
}
}
/**
* Can block if buffer is full
*/
@Override
protected void sendToNode(Message message, short node, InetSocketAddress address) {
try {
if (LOG.isDebugEnabled())
LOG.debug("Sending to node {} ({}): {}", new Object[]{node, address, message});
message.cloneDataBuffers(); // important, as we're going to be doing actual sending on another thread
final NodePeer peer = peers.get(node);
if (peer == null)
throw new NodeNotFoundException(node);
peer.sendMessage(message);
executor.submit(peer);
} catch (InterruptedException ex) {
LOG.error("InterruptedException", ex);
throw new RuntimeException(ex);
} catch (Exception ex) {
LOG.error("Error while sending message " + message + " to node " + node, ex);
}
}
/**
* Can block
*/
@Override
protected synchronized void broadcast(Message message) { // synchronized for message ID ordering
try {
assert message.isBroadcast() && !message.isResponse();
assignMessageId(message);
final boolean unicast = getNumPeerNodes() < minimumNodesToMulticast;
final ShortSet nodes = new ShortOpenHashSet();
for (NodePeer peer : peers.values()) {
nodes.add(peer.node);
peer.sendMessage(message, unicast);
executor.submit(peer);
}
if (nodes.isEmpty()) {
if (message instanceof LineMessage) {
LOG.debug("No other nodes in cluster. Responding with NOT_FOUND to message {}", message);
receive(Message.NOT_FOUND((LineMessage) message).setIncoming());
}
return;
}
broadcastPeer.sendMessage(message, nodes, unicast);
if (!unicast)
executor.submit(broadcastPeer);
} catch (InterruptedException ex) {
LOG.error("InterruptedException", ex);
throw new RuntimeException(ex);
}
}
// visible for testing
void messageReceived(MessagePacket packet) {
if (!getCluster().isMaster())
return;
LOG.debug("Received packet {}", packet);
final long now = System.nanoTime();
packet.setTimestamp(now);
final short node = packet.getNode();
final NodePeer peer = peers.get(node);
if (peer == null)
throw new RuntimeException("Message received from unhandled node " + node);
try {
peer.receivePacket(packet); // we're now running in the executor we installed in the netty pipeline.
} catch (InterruptedException ex) {
LOG.error("InterruptedException", ex);
throw new RuntimeException(ex);
}
}
@Override
public synchronized void nodeAdded(short id) {
super.nodeAdded(id);
if (id == 0)
return;
if (peers.get(id) != null)
return;
final NodePeer peer = new NodePeer(id);
LOG.info("Adding peer {} for node {}", peer, id);
peer.setAddress(getNodeAddress(id));
peers.put(id, peer);
}
@Override
public synchronized void nodeSwitched(short id) {
super.nodeSwitched(id);
final NodePeer peer = peers.get(id);
LOG.info("Node switched. Fixing peer {}", peer);
peer.setAddress(getNodeAddress(id));
executor.submit(peer); // resend
executor.submit(broadcastPeer); // resend
}
@Override
public synchronized void nodeRemoved(short id) {
super.nodeRemoved(id);
final NodePeer peer = peers.get(id);
if (peer != null)
peer.removed();
peers.remove(id);
broadcastPeer.removeNode(id);
}
private static final ThreadLocal<Boolean> recursive = new ThreadLocal<Boolean>();
abstract class Peer implements Callable<Void> {
protected final ArrayBlockingQueue<Message> queue = new ArrayBlockingQueue<Message>(maxQueueSize);
protected Message overflow;
protected MessagePacket sentPacket;
private int delayMultiplier = 1;
private long lastSent;
private long nextSend;
private final Set<Message> timeouts = Collections.newSetFromMap(new ConcurrentHashMap<Message, Boolean>());
private long lastTimeoutsCleanup;
/**
* This can block!
*/
public void sendMessage(Message message) throws InterruptedException {
if (!queue.offer(message)) {
LOG.info("Adding message {} to full queue. Waiting for available space.", message);
LOG.warn("no space in Peer {}", this);
if (recursive.get() == Boolean.TRUE) {
LOG.error("Queue is too small");
throw new RuntimeException("Queue full");
}
queue.put(message);
}
}
public int getQueueLength() {
return queue.size();
}
protected void forceResend() {
this.lastSent = 0;
this.nextSend = 0;
this.delayMultiplier = 0;
}
protected boolean isTimeToResned(long now) {
if (now > nextSend) {
nextSend = Long.MAX_VALUE;
lastSent = now;
return true;
} else
return false;
}
protected void resendIn(long now, long delay) {
if (LOG.isDebugEnabled())
LOG.debug("Peer {} rescheduling in {}", this, delay);
nextSend = now + delay;
executor.schedule(this, delay, NANOSECONDS);
}
protected void resend(long now) {
long delay = resendPeriodNanos << delayMultiplier;
if (exponentialBackoff)
delayMultiplier++;
if (jitter)
delay = randInterval(delay);
resendIn(now, delay);
}
protected long getLastSent() {
return lastSent;
}
protected void addTimeout(Message message) {
timeouts.add(message);
}
protected boolean isTimeout(Message response) {
return timeouts.remove(response);
}
protected synchronized void cleanupTimeouts(long now) {
if (now - lastTimeoutsCleanup >= NANOSECONDS.convert(10, SECONDS)) {
for (Iterator<Message> it = timeouts.iterator(); it.hasNext();) {
if (now - it.next().getTimestamp() >= NANOSECONDS.convert(10, SECONDS))
it.remove();
}
lastTimeoutsCleanup = now;
}
}
}
class NodePeer extends Peer {
public final short node;
private volatile boolean removed = false;
private InetSocketAddress nodeAddress;
private boolean hasRequests = false; // true if not all messages in the sent packet are responses
private boolean requestsOnly = true; // true if none of the messages in the sent packet are responses
private volatile boolean broadcast; // true if the sent packet contains a (single) broadcast (and only that)
private final LongSet pendingRequests = new LongOpenHashSet();
private final Set<Message> unicastBroadcasts = Collections.newSetFromMap(new ConcurrentHashMap<Message, Boolean>());
private long lastReceivedBroadcastId;
public NodePeer(short node) {
this.node = node;
}
public synchronized void setAddress(InetSocketAddress nodeAddress) {
LOG.info("Node peer {} set address to {}", this, nodeAddress);
this.nodeAddress = nodeAddress;
lastReceivedBroadcastId = 0;
if (sentPacket != null) {
for (Iterator<Message> it = sentPacket.iterator(); it.hasNext();) {
final Message message = it.next();
if (message.isResponse()) {
LOG.debug("Peer {} removing response {} because of node switch.", this, message);
it.remove(); // if our peer hasn't requested again then it must have received our response
}
}
}
forceResend();
}
@Override
public synchronized String toString() {
return "NodePeer{" + "node=" + node + ", nodeAddress=" + nodeAddress + ", lastSent=" + getLastSent() + ", sentPacket=" + sentPacket + ", pendingRequests=" + pendingRequests + ", next=" + overflow + ", queue=" + queue + ", broadcast=" + broadcast + '}';
}
public boolean isBroadcast() {
return broadcast;
}
public void unicastBroadcast() {
assert broadcast;
LOG.debug("Node peer {} is asked to unicast broadcast.", this);
broadcast = false;
}
public void removed() {
removed = true;
}
@Override
public void sendMessage(Message message) throws InterruptedException {
synchronized (queue) { // syncrhonization ensures message id is in the order of messages put in the queue
assignMessageId(message);
super.sendMessage(message);
}
}
public void sendMessage(Message message, boolean unicastBroadcast) throws InterruptedException {
if (unicastBroadcast && message.isBroadcast())
unicastBroadcasts.add(message);
sendMessage(message);
}
void receivePacket(MessagePacket packet) throws InterruptedException {
final List<Message> received = new ArrayList<Message>(packet.numMessages());
final List<Message> broadcastResponses = new ArrayList<Message>(packet.numMessages());
synchronized (this) {
handleReceived(packet, received, broadcastResponses);
}
for (Message message : broadcastResponses)
broadcastPeer.receivedResponse(message, received);
recursive.set(Boolean.TRUE);
try {
for (Message message : received) {
LOG.debug("Passing received message {} to cache", message);
receive(message); // XXXX
}
} finally {
recursive.remove();
}
call();
}
@Override
public Void call() throws InterruptedException {
if (recursive.get() == Boolean.TRUE)
return null;
recursive.set(Boolean.TRUE);
try {
if (removed || getCluster().getMaster(node) == null) {
LOG.debug("Node removed from the cluster so returning from peer {}", this);
return null; // don't reschedule
}
final List<Message> received = new ArrayList<Message>();
synchronized (this) {
LOG.trace("Peer {} CALL", this);
final long now = System.nanoTime();
handleTimeout(now, received);
handleQueue(now);
if (sentPacket != null && sentPacket.isEmpty())
sentPacket = null;
if (sentPacket != null && !broadcast) {
if (isTimeToResned(now)) { // if messages have been added to sentPacket has changed, handleQueue sets lastSent to 0
LOG.debug("Peer {} sending packet {}", this, sentPacket);
channel.write(sentPacket, nodeAddress);
if (hasRequests)
resend(now);
}
}
}
for (Message message : received)
receive(message);
LOG.trace("Peer {} CALL DONE", this);
return null;
} finally {
recursive.remove();
}
}
private void handleReceived(MessagePacket receivedPacket, List<Message> received, List<Message> broadcastResponses) {
if (receivedPacket == null)
return;
LOG.debug("Peer {} has received packet {}", this, receivedPacket);
boolean oobMulticast = false;
if (receivedPacket.isMulticast()) { // multicast messages may overlap with unicast ones if the original broadcast was sent as a unicast, say if the peers sentPacket wasn't empty
long maxIdInPacket = -1;
for (Iterator<Message> it = receivedPacket.iterator(); it.hasNext();) {
final Message message = it.next();
// if (message.getMessageId() < lastReceivedBroadcastId) {
// LOG.trace("Peer {} received a multicast message {} which has already been seen.", this, message);
// it.remove();
// }
maxIdInPacket = Math.max(maxIdInPacket, message.getMessageId());
}
if (maxIdInPacket < lastReceivedBroadcastId) {
LOG.debug("Peer {} received an out-of-band multicast packet {} which has already been seen.", this, receivedPacket);
oobMulticast = true;
}
}
if (receivedPacket.isEmpty())
return;
if (!oobMulticast && sentPacket != null) {
for (Iterator<Message> it = sentPacket.iterator(); it.hasNext();) {
final Message message = it.next();
// here we rely on Message.equals() to match request/response
if (message.isResponse() && !receivedPacket.contains(message)) {
LOG.debug("Peer {} removing response {} from sent packet because it was no longer asked for.", this, message);
it.remove(); // if our peer hasn't requested again then it must have received our response
}
}
}
for (Message message : receivedPacket) {
message.setTimestamp(receivedPacket.getTimestamp());
if (message.isBroadcast()) {
if (message.getMessageId() > lastReceivedBroadcastId)
lastReceivedBroadcastId = message.getMessageId();
}
// here we rely on Message.equals() to match request/response
if (message.isResponse()) {
final Message request = (sentPacket != null ? sentPacket.getMessage(message) : null);
if (request == null && !(isTimeout(message) || (broadcast && broadcastPeer.isTimeout(message)))) {
LOG.debug("Peer {} ignoring repeat response {}", this, message);
continue; // we may be re-receiving the response, so the request may be gone. in this case we don't need to pass the message again to the receiver
}
if (LOG.isDebugEnabled())
LOG.debug("Peer {} received response {} for request ({})", new Object[]{this, message, request != null ? request : "TIMEOUT"});
if (request != null) {
if (request.isBroadcast())
broadcastResponses.add(message);
// if(message.getType() == Message.Type.CHNGD_OWNR && ((Message.CHNGD_OWNR)message).getNewOwner() == message.getNode()) {
// // this is a quickReplyToBroadcast
// // TODO
// }
sentPacket.removeMessage(message);
}
} else {
if (sentPacket != null && sentPacket.contains(message)) {
LOG.debug("Peer {} already has a response for message {}", this, message);
continue; // no need to re-generate a response we already have
}
if (pendingRequests.contains(message.getMessageId())) {
LOG.debug("Peer {} already has a request pending for message {}", this, message);
continue; // we don't pass on requests to the receiver more than once
} else
pendingRequests.add(message.getMessageId());
}
if (message.getType() == Message.Type.ACK)
continue; // we do not pass ACKs on to the receiver
received.add(message); // getReceiver().receive(message);
if (!message.isResponse() && !message.isReplyRequired()) {
if (!queue.offer(Message.ACK(message))) {
LOG.error("Queue capacity for perr {} exceeded", this);
throw new RuntimeException("Peer queue full!");
}
}
}
//receivedPacket = null;
if (sentPacket != null) {
forceResend();
if (sentPacket.isEmpty()) {
sentPacket = null;
broadcast = false;
hasRequests = false;
requestsOnly = true;
} else {
// update hasRequests, requestsOnly and broadcast
boolean _hasRequests = false;
boolean _requestsOnly = true;
boolean _broadcast = true;
for (Message message : sentPacket) {
if (message.isResponse())
_requestsOnly = false;
else
_hasRequests = true;
if (!message.isBroadcast())
_broadcast = false;
}
hasRequests = _hasRequests;
requestsOnly = _requestsOnly;
if (!broadcast && _broadcast) {
LOG.trace("Peer {} notifying broadcast.", this);
executor.submit(broadcastPeer);
}
broadcast = _broadcast;
}
}
}
private void handleTimeout(long now, List<Message> received) {
if (broadcast || sentPacket == null || sentPacket.isEmpty())
return;
final long timeoutNanos = NANOSECONDS.convert(getTimeout(), MILLISECONDS);
for (Iterator<Message> it = sentPacket.reverseIterator(); it.hasNext();) {
final Message message = it.next();
if (message.getType() != Message.Type.INV && now - message.getTimestamp() > timeoutNanos) {
if (message.isResponse() || message.isBroadcast())
continue;
if (message instanceof LineMessage) {
LOG.debug("Timeout on message {}", message);
received.add(Message.TIMEOUT((LineMessage) message).setIncoming());
}
it.remove();
addTimeout(message);
} else
break;
}
if (sentPacket.isEmpty()) {
sentPacket = null;
broadcast = false;
hasRequests = false;
requestsOnly = true;
}
cleanupTimeouts(now);
}
/**
* Specifies that a message should not be resent, but a response is still possible
*
* @param message
*/
public synchronized void markAsTimeout(Message message) {
if (sentPacket.removeMessage(message.getMessageId()))
addTimeout(message);
}
private synchronized void handleQueue(long start) throws InterruptedException {
// ProbLem:
// assume we send a full packet with requests only, and our peer send us a full packet with requests only.
// we cannot add requests to the sentPacket b/c it's full, so we must wait for our peer to respond so that we can emty
// the packet, only it can't b/c its sentPacket is also full - we got a deadlock.
// as I see it, the only way to truly resolve it is to have multi-part packets, but we don't want to do that.
// what we do is that we don't allow a packet with requests only to be full - we always leave room for a response.
// assumes hasRequests and requestsOnly are up to date.
Message next = overflow;
overflow = null;
if (next == null)
next = queue.poll();
for (;;) {
LOG.trace("handleQueue loop");
if (next == null) {
LOG.trace("handleQueue loop: next == null");
break;
}
overflow = next; // we put the next message into overflow. if we _don't_ break out of the loop and use the message, we'll null overflow
final boolean unicastBroadcast = next.isBroadcast() && unicastBroadcasts.remove(next);
if (broadcast && (!next.isBroadcast() || unicastBroadcast)) {
LOG.trace("Node peer {} not taking non-broadcast message {} during broadcast", this, next);
break; // we're not taking any non-broadcast messages during broadcast
}
if (!broadcast && next.isBroadcast() && !unicastBroadcast) {
if (sentPacket == null || sentPacket.isEmpty()) {
LOG.debug("Node peer {} going into broadcast mode for message {}.", this, next);
broadcast = true;
}
// else, we add message to packet, and continue transmitting.
// if the packet had responses only, the new broadcast request would force a re-send and expedite matters
// if a response for the broadcast is received before we get a chance to multicast, that's ok because we simply remove the node
// from the BroadcastEntry
}
if (next.size() > maxPacketSize) {
LOG.error("Message {} is larger than the maximum packet size {}", next, maxPacketSize);
throw new RuntimeException("Message is larger than maxPacketSize");
}
if (next.size() + sentPacketSizeInBytes() > maxPacketSize) {
if (next.isResponse() && requestsOnly)
LOG.warn("IMPORTANT: Response message {} does not fit in packet {} which contains only requests. THIS MAY CAUSE A DEADLOCK!", next, sentPacket);
LOG.debug("Message {} cannot be added to packet now; packet full (size = {})", next, next.size());
break;
}
if (!next.isResponse()) {
if (requestsOnly && next.size() + sentPacketSizeInBytes() > maxRequestOnlyPacketSize && sentPacketSizeInBytes() > 0) {
// check if packet consists of requestOnly message unless it is only one message.
LOG.debug("NOT Sending requests only {}. can't add to packet {} bytes long.", next, sentPacketSizeInBytes());
break;
}
hasRequests = true;
} else
requestsOnly = false;
if (next.isResponse())
pendingRequests.remove(next.getMessageId());
LOG.debug("Adding message {} to sent-packet", next);
if (sentPacket == null)
sentPacket = new MessagePacket();
sentPacket.addMessage(next);
forceResend();
overflow = null;
if (broadcast) {
LOG.trace("Peer {} notifying broadcast.", this);
executor.submit(broadcastPeer);
}
final long now = System.nanoTime();
if ((now - start + minDelayNanos) > maxDelayNanos)
break;
next = queue.poll(minDelayNanos, NANOSECONDS);
}
}
private int sentPacketSizeInBytes() {
return sentPacket != null ? sentPacket.sizeInBytes() : 0;
}
}
class BroadcastPeer extends Peer {
private final ConcurrentMap<Long, BroadcastEntry> broadcasts = new ConcurrentHashMap<Long, BroadcastEntry>();
@Override
public String toString() {
return "BroadcastPeer{" + "multicastAddress=" + multicastGroup + ", lastSent=" + getLastSent() + ", sentPacket=" + sentPacket + ", next=" + overflow + ", queue=" + queue + '}';
}
public void sendMessage(Message message, ShortSet nodes, boolean unicast) throws InterruptedException {
broadcasts.put(message.getMessageId(), new BroadcastEntry(message, nodes));
if (!unicast)
sendMessage(message);
}
@Override
public Void call() throws InterruptedException {
final List<Message> received = new ArrayList<Message>();
synchronized (this) {
LOG.trace("BroadcastPeer CALL");
final long now = System.nanoTime();
handleTimeout(now, received);
handleQueue(now);
if (sentPacket != null && sentPacket.isEmpty())
sentPacket = null;
if (isTimeToResned(now)) {
if (sentPacket != null) { // if messages have been added tos sentPacket has changed, handleQueue sets lastSent to 0
assert !sendToServerInsteadOfMulticast;
LOG.debug("BroadcastPeer {} multicasting packet {}", this, sentPacket);
channel.write(sentPacket, multicastGroup);
resend(now);
} else if (!broadcasts.isEmpty()) {
executor.schedule(this, getTimeout(), MILLISECONDS);
}
}
}
for (Message message : received)
receive(message);
LOG.trace("BroadcastPeer CALL DONE");
return null;
}
private void handleQueue(long start) throws InterruptedException {
Message next = overflow;
overflow = null;
if (next == null)
next = queue.poll();
loop:
for (;;) {
if (next == null)
break;
overflow = next; // we put the next message into overflow. if we _don't_ break out of the loop and use the message, we'll null overflow
if (next.size() > maxPacketSize) {
LOG.error("Message {} is larger than the maximum packet size {}", next, maxPacketSize);
throw new RuntimeException("Message is larger than maxPacketSize");
}
if (sentPacket != null && next.size() + sentPacket.sizeInBytes() > maxPacketSize)
break;
LOG.debug("Waiting for peers to enter broadcast mode for message {}", next);
BroadcastEntry entry = broadcasts.get(next.getMessageId());
if (entry != null) {
if (entry.nodes.isEmpty()) {
broadcasts.remove(next.getMessageId());
if (next instanceof LineMessage) {
LOG.debug("No other nodes in cluster. Responding with NOT_FOUND to message {}", next);
receive(Message.NOT_FOUND((LineMessage) next).setIncoming());
}
entry = null;
}
}
if (entry != null) {
for (ShortIterator it = entry.nodes.iterator(); it.hasNext();) {
final short node = it.next();
final NodePeer peer = peers.get(node);
synchronized (peer) {
if (!(peer.isBroadcast() && peer.sentPacket.contains(next.getMessageId()))) {
LOG.trace("Waiting for peer {}.", peer);
break loop;
}
LOG.trace("Peer {} ok (broadcast {})", peer, next);
}
}
LOG.debug("Adding message {} to sent-packet", next);
if (sentPacket == null)
sentPacket = new MessagePacket();
sentPacket.addMessage(next);
forceResend();
}
overflow = null;
final long now = System.nanoTime();
if (maxDelayNanos > (now - start + minDelayNanos))
break;
next = queue.poll(minDelayNanos, NANOSECONDS);
}
}
private void handleTimeout(long now, List<Message> received) {
if (broadcasts.isEmpty())
return;
final long timeoutNanos = NANOSECONDS.convert(getTimeout(), MILLISECONDS);
for (Iterator<BroadcastEntry> it = broadcasts.values().iterator(); it.hasNext();) {
final BroadcastEntry entry = it.next();
final Message message = entry.message;
if (message.getType() != Message.Type.INV && now - message.getTimestamp() > timeoutNanos) {
if (message instanceof LineMessage) {
LOG.debug("Timeout on message {}", message);
received.add(Message.TIMEOUT((LineMessage) message).setIncoming());
}
it.remove();
releasePeers(entry, (short) -1);
addTimeout(message);
if (sentPacket != null)
sentPacket.removeMessage(message.getMessageId());
}
}
if (sentPacket != null && sentPacket.isEmpty())
sentPacket = null;
cleanupTimeouts(now);
}
public void receivedResponse(Message message, List<Message> received) {
final BroadcastEntry entry = broadcasts.get(message.getMessageId());
if (entry == null)
return;
synchronized (this) {
boolean done = entry.removeNode(message.getNode());
if (message.getType() != Message.Type.ACK) {// this is a response - no need to wait for further acks
LOG.debug("Message {} is a reply to a broadcast! (discarding pending)", message);
if (!done)
releasePeers(entry, message.getNode());
done = true;
} else {
if (LOG.isDebugEnabled())
LOG.debug("Got ACK from {} to message {}", message.getNode(), entry.message);
final int numNodes = entry.nodes.size();
if (done) {
if (entry.message instanceof LineMessage) {
LOG.debug("Got all ACKs for message {}, but no response - sending NOT_FOUND to cache!", entry.message);
received.add(Message.NOT_FOUND((LineMessage) entry.message).setIncoming());
}
} else if (numNodes < minimumNodesToMulticast && (numNodes + 1) >= minimumNodesToMulticast) {
if (sentPacket != null)
sentPacket.removeMessage(message.getMessageId()); // don't multicast...
// unicast:
final long now = System.nanoTime();
final long sinceLastSent = now - getLastSent();
long delay = resendPeriodNanos - sinceLastSent;
delay = (delay >= 0 ? delay : 0);
for (ShortIterator it = entry.nodes.iterator(); it.hasNext();) {
final NodePeer peer = peers.get(it.next());
if (peer.isBroadcast()) {
peer.unicastBroadcast();
peer.forceResend();
peer.resendIn(now, delay);
executor.submit(peer);
}
}
}
}
if (done) {
if (sentPacket != null)
sentPacket.removeMessage(message.getMessageId());
broadcasts.remove(message.getMessageId());
}
if (sentPacket != null && sentPacket.isEmpty())
sentPacket = null;
}
}
private void releasePeers(BroadcastEntry entry, short node) {
final Message message = entry.message;
for (ShortIterator it = entry.nodes.iterator(); it.hasNext();) {
final NodePeer peer = peers.get(it.next());
if (peer.isBroadcast()) {
LOG.debug("Broadcast releasing peer {} for message {}", peer, message);
if (peer.node != node) {
LOG.debug("Broadcast marking message {} as timeout for peer {}", message, peer);
peer.markAsTimeout(message);
}
peer.unicastBroadcast();
executor.submit(peer);
}
}
}
public void removeNode(short node) {
synchronized (this) {
for (Iterator<Map.Entry<Long, BroadcastEntry>> it = broadcasts.entrySet().iterator(); it.hasNext();) {
BroadcastEntry entry = it.next().getValue();
if (entry.removeNode(node) && entry.message instanceof LineMessage) {
LOG.debug("Got all ACKs for message {}, but no response - sending NOT_FOUND to cache!", entry.message);
receive(Message.NOT_FOUND((LineMessage) entry.message).setIncoming());
it.remove();
}
}
}
}
}
private static class BroadcastEntry {
final Message message;
final ShortSet nodes;
public BroadcastEntry(Message message, ShortSet nodes) {
this.message = message;
this.nodes = nodes;
this.nodes.remove(Comm.SERVER); // NOT TO SERVER
LOG.debug("Awaiting ACKS for message {} from nodes {}", message, this.nodes);
}
public synchronized void addNode(short node) {
nodes.add(node);
}
public synchronized boolean removeNode(short node) {
nodes.remove(node);
return nodes.isEmpty();
}
}
private int getNumPeerNodes() {
return getCluster().getNodes().size() - (getCluster().getNodes().contains(Comm.SERVER) ? 1 : 0) + 1;
}
private static long randInterval(long expected) {
return (long) randExp(1.0 / expected);
}
/**
* Return a real number from an exponential distribution with rate lambda. Based on
* http://en.wikipedia.org/wiki/Inverse_transform_sampling
*/
private static double randExp(double lambda) {
return -Math.log(1 - ThreadLocalRandom.current().nextDouble()) / lambda;
}
BroadcastPeer getBroadcastPeer() {
return broadcastPeer;
}
ConcurrentMap<Short, NodePeer> getPeers() {
return peers;
}
}