package water;
import java.nio.channels.DatagramChannel;
import java.util.Date;
import java.util.Random;
import water.util.Log;
/**
* The Thread that looks for UDP Cloud requests.
*
* This thread just spins on reading UDP packets from the kernel and either
* dispatching on them directly itself (if the request is known short) or
* queuing them up for worker threads.
* @author <a href="mailto:cliffc@h2o.ai"></a>
* @version 1.0
*/
public class UDPReceiverThread extends Thread {
static private int _unknown_packets_per_sec = 0;
static private long _unknown_packet_time = 0;
static final Random RANDOM_UDP_DROP = new Random();
private final DatagramChannel datagramChannel;
public UDPReceiverThread(DatagramChannel datagramChannel) {
super("D-UDP-Recv");
this.datagramChannel = datagramChannel;
}
// ---
// Started by main() on a single thread, this code manages reading UDP packets
@SuppressWarnings("resource")
public void run() {
Thread.currentThread().setPriority(Thread.MAX_PRIORITY-1);
DatagramChannel sock = datagramChannel, errsock = null;
boolean saw_error = false;
while( true ) {
try {
// Cleanup from any prior socket failures. Rare unless we're really sick.
if( errsock != null ) { // One time attempt a socket close
final DatagramChannel tmp2 = errsock; errsock = null;
tmp2.close(); // Could throw, but errsock cleared for next pass
}
if( saw_error ) Thread.sleep(1000); // prevent deny-of-service endless socket-creates
saw_error = false;
// ---
// Common-case setup of a socket
if( sock == null ) {
sock = DatagramChannel.open();
sock.socket().bind(H2O.SELF._key);
}
// Receive a packet & handle it
basic_packet_handling(new AutoBuffer(sock));
} catch( java.nio.channels.AsynchronousCloseException ex ) {
break; // Socket closed for shutdown
} catch( java.nio.channels.ClosedChannelException ex ) {
break; // Socket closed for shutdown
} catch( Exception e ) {
// On any error from anybody, close all sockets & re-open
Log.err("UDP Receiver error on port "+H2O.H2O_PORT,e);
saw_error = true;
errsock = sock ;
sock = null; // Signal error recovery on the next loop
}
}
}
// Basic packet handling:
// - Timeline record it
static public void basic_packet_handling( AutoBuffer ab ) throws java.io.IOException {
// Randomly drop 1/10th of the packets, as-if broken network. Dropped
// packets are timeline recorded before dropping - and we still will
// respond to timelines and suicide packets.
int drop = H2O.ARGS.random_udp_drop &&
RANDOM_UDP_DROP.nextInt(5) == 0 ? 2 : 0;
// Record the last time we heard from any given Node
TimeLine.record_recv(ab, false, drop);
final long now = ab._h2o._last_heard_from = System.currentTimeMillis();
// Snapshots are handled *IN THIS THREAD*, to prevent more UDP packets from
// being handled during the dump. Also works for packets from outside the
// Cloud... because we use Timelines to diagnose Paxos failures.
int ctrl = ab.getCtrl();
ab.getPort(); // skip the port bytes
if( ctrl == UDP.udp.timeline.ordinal() ) {
UDP.udp.timeline._udp.call(ab);
return;
}
// Suicide packet? Short-n-sweet...
if( ctrl == UDP.udp.rebooted.ordinal())
UDPRebooted.checkForSuicide(ctrl, ab);
// Drop the packet.
if( drop != 0 ) return;
// Get the Cloud we are operating under for this packet
H2O cloud = H2O.CLOUD;
// Check cloud membership; stale ex-members are "fail-stop" - we mostly
// ignore packets from them (except paxos packets).
boolean is_member = cloud.contains(ab._h2o);
boolean is_client = ab._h2o._heartbeat._client;
// Some non-Paxos packet from a non-member. Probably should record & complain.
// Filter unknown-packet-reports. In bad situations of poisoned Paxos
// voting we can get a LOT of these packets/sec, flooding the logs.
if( !(UDP.udp.UDPS[ctrl]._paxos || is_member || is_client) ) {
_unknown_packets_per_sec++;
long timediff = ab._h2o._last_heard_from - _unknown_packet_time;
if( timediff > 1000 ) {
// If this is a recently booted client node... coming up right after a
// prior client was shutdown, it might see leftover trash UDP packets
// from the servers intended for the prior client.
if( !(H2O.ARGS.client && now-H2O.START_TIME_MILLIS.get() < HeartBeatThread.CLIENT_TIMEOUT) )
Log.warn("UDP packets from outside the cloud: "+_unknown_packets_per_sec+"/sec, last one from "+ab._h2o+ " @ "+new Date());
_unknown_packets_per_sec = 0;
_unknown_packet_time = ab._h2o._last_heard_from;
}
ab.close();
return;
}
// Paxos stateless packets & ACKs just fire immediately in a worker
// thread. Dups are handled by these packet handlers directly. No
// current membership check required for Paxos packets.
//
// Handle the case of packet flooding draining all the available
// ByteBuffers and running the JVM out of *native* memory, triggering
// either a large RSS (and having YARN kill us for being over-budget) or
// simply tossing a OOM - but a out-of-native-memory nothing to do with
// heap memory.
//
// All UDP packets at this stage have fairly short lifetimes - Exec packets
// (which you might think to be unboundedly slow) are actually just going
// through the deserialization call in RPC.remote_exec - and the deser'd
// DTask gets tossed on a low priority queue to do "the real work". Since
// this is coming from a UDP packet the deser work is actually small.
H2O.submitTask(new FJPacket(ab,ctrl));
}
}