package lbms.plugins.mldht.kad;
import java.io.File;
import java.io.IOException;
import java.net.*;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.*;
import lbms.plugins.mldht.kad.messages.AnnounceRequest;
import lbms.plugins.mldht.kad.messages.AnnounceResponse;
import lbms.plugins.mldht.kad.messages.ErrorMessage;
import lbms.plugins.mldht.kad.messages.FindNodeRequest;
import lbms.plugins.mldht.kad.messages.FindNodeResponse;
import lbms.plugins.mldht.kad.messages.GetPeersRequest;
import lbms.plugins.mldht.kad.messages.GetPeersResponse;
import lbms.plugins.mldht.kad.messages.MessageBase;
import lbms.plugins.mldht.kad.messages.PingRequest;
import lbms.plugins.mldht.kad.messages.PingResponse;
import lbms.plugins.mldht.kad.messages.ErrorMessage.ErrorCode;
/**
* @author Damokles
*
*/
public class DHT implements DHTBase {
public static enum DHTtype {
IPV4_DHT("IPv4",20+4+2, 4+2, Inet4Address.class),
IPV6_DHT("IPv6",20+16+2, 16+2, Inet6Address.class);
public final int NODES_ENTRY_LENGTH;
public final int ADDRESS_ENTRY_LENGTH;
public final Class<? extends InetAddress> PREFERRED_ADDRESS_TYPE;
public final String shortName;
private DHTtype(String shortName, int nodeslength, int addresslength, Class<? extends InetAddress> addresstype) {
this.shortName = shortName;
this.NODES_ENTRY_LENGTH = nodeslength;
this.PREFERRED_ADDRESS_TYPE = addresstype;
this.ADDRESS_ENTRY_LENGTH = addresslength;
}
}
private static DHTLogger logger;
private static LogLevel logLevel = LogLevel.Info;
private static ScheduledThreadPoolExecutor scheduler;
private static ThreadGroup executorGroup;
static {
executorGroup = new ThreadGroup("mlDHT");
scheduler = new ScheduledThreadPoolExecutor(2, new ThreadFactory() {
public Thread newThread (Runnable r) {
Thread t = new Thread(executorGroup, r, "mlDHT Executor");
t.setDaemon(true);
return t;
}
});
scheduler.setKeepAliveTime(20, TimeUnit.SECONDS);
scheduler.allowCoreThreadTimeOut(true);
logger = new DHTLogger() {
public void log (String message) {
// System.out.println(message);
};
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTLogger#log(java.lang.Exception)
*/
public void log (Exception e) {
e.printStackTrace();
}
};
}
private boolean running;
private boolean bootstrapping;
private long lastBootstrap;
private long lastRandomLookup;
private int port;
private Node node;
private RPCServer srv;
private Database db;
private TaskManager tman;
private ExpireTimer expire_timer;
private File table_file;
private boolean useRouterBootstrapping;
private List<DHTStatsListener> statsListeners;
private List<DHTStatusListener> statusListeners;
private DHTStats stats;
private DHTStatus status;
private PopulationEstimator estimator;
private final DHTtype type;
private List<ScheduledFuture<?>> scheduledActions = new ArrayList<ScheduledFuture<?>>();
static Map<DHTtype,DHT> dhts;
public synchronized static Map<DHTtype, DHT> createDHTs() {
if(dhts == null)
{
dhts = new EnumMap<DHTtype,DHT>(DHTtype.class);
dhts.put(DHTtype.IPV4_DHT, new DHT(DHTtype.IPV4_DHT));
dhts.put(DHTtype.IPV6_DHT, new DHT(DHTtype.IPV6_DHT));
}
return dhts;
}
public static DHT getDHT(DHTtype type)
{
return dhts.get(type);
}
private DHT(DHTtype type) {
this.type = type;
expire_timer = new ExpireTimer();
stats = new DHTStats();
status = DHTStatus.Stopped;
statsListeners = new ArrayList<DHTStatsListener>(2);
statusListeners = new ArrayList<DHTStatusListener>(2);
estimator = new PopulationEstimator();
}
public void ping (PingRequest r) {
if (!running) {
return;
}
// ignore requests we get from ourself
if (r.getID().equals(node.getOurID())) {
return;
}
PingResponse rsp = new PingResponse(r.getMTID(), node.getOurID());
rsp.setOrigin(r.getOrigin());
srv.sendMessage(rsp);
node.recieved(this, r);
}
public void findNode (FindNodeRequest r) {
if (!running) {
return;
}
// ignore requests we get from ourself
if (r.getID().equals(node.getOurID())) {
return;
}
node.recieved(this, r);
// find the K closest nodes and pack them
KClosestNodesSearch kns4 = new KClosestNodesSearch(r.getTarget(), DHTConstants.MAX_ENTRIES_PER_BUCKET, getDHT(DHTtype.IPV4_DHT));
KClosestNodesSearch kns6 = new KClosestNodesSearch(r.getTarget(), DHTConstants.MAX_ENTRIES_PER_BUCKET, getDHT(DHTtype.IPV6_DHT));
// add our local address of the respective DHT for cross-seeding, but not for local requests
kns4.fill(DHTtype.IPV4_DHT != type);
kns6.fill(DHTtype.IPV6_DHT != type);
FindNodeResponse fnr = new FindNodeResponse(r.getMTID(), node
.getOurID(), r.doesWant4() ? kns4.pack() : null,r.doesWant6() ? kns6.pack() : null);
fnr.setOrigin(r.getOrigin());
srv.sendMessage(fnr);
}
public void response (MessageBase r) {
if (!running) {
return;
}
node.recieved(this, r);
}
public void getPeers (GetPeersRequest r) {
if (!running) {
return;
}
// ignore requests we get from ourself
if (r.getID().equals(node.getOurID())) {
return;
}
node.recieved(this, r);
List<DBItem> dbl = new ArrayList<DBItem>();
db.sample(r.getInfoHash(), dbl, 50,type);
// generate a token
ByteWrapper token = db.genToken(r.getOrigin().getAddress(), r
.getOrigin().getPort(), r.getInfoHash());
KClosestNodesSearch kns4 = new KClosestNodesSearch(r.getInfoHash(),DHTConstants.MAX_ENTRIES_PER_BUCKET,getDHT(DHTtype.IPV4_DHT));
KClosestNodesSearch kns6 = new KClosestNodesSearch(r.getInfoHash(),DHTConstants.MAX_ENTRIES_PER_BUCKET,getDHT(DHTtype.IPV6_DHT));
// add our local address of the respective DHT for cross-seeding, but not for local requests
kns4.fill(DHTtype.IPV4_DHT != type);
kns6.fill(DHTtype.IPV6_DHT != type);
GetPeersResponse resp = new GetPeersResponse(r.getMTID(), node.getOurID(), r.doesWant4() ? kns4.pack() : null, r.doesWant6() ? kns6.pack() : null, token.arr);
resp.setPeerItems(dbl);
resp.setDestination(r.getOrigin());
srv.sendMessage(resp);
}
public void announce (AnnounceRequest r) {
if (!running) {
return;
}
// ignore requests we get from ourself
if (r.getID().equals(node.getOurID())) {
return;
}
node.recieved(this, r);
// first check if the token is OK
ByteWrapper token = new ByteWrapper(r.getToken());
if (!db.checkToken(token, r.getOrigin().getAddress(), r
.getOrigin().getPort(), r.getInfoHash())) {
logDebug("DHT Received Announce Request with invalid token.");
sendError(r, ErrorCode.ProtocolError.code, "Invalid Token");
return;
}
logDebug("DHT Received Announce Request, adding peer to db: "
+ r.getOrigin().getAddress());
// everything OK, so store the value
byte[] tdata = new byte[r.getOrigin().getAddress().getAddress().length + 2];
ByteBuffer bb = ByteBuffer.wrap(tdata);
bb.put(r.getOrigin().getAddress().getAddress());
bb.putShort((short) r.getOrigin().getPort());
db.store(r.getInfoHash(), new DBItem(tdata));
// send a proper response to indicate everything is OK
AnnounceResponse rsp = new AnnounceResponse(r.getMTID(), node
.getOurID());
rsp.setOrigin(r.getOrigin());
srv.sendMessage(rsp);
}
public void error (ErrorMessage r) {
DHT.logError("Error [" + r.getCode() + "] from: " + r.getOrigin()
+ " Message: \"" + r.getMessage() + "\"");
}
public void timeout (MessageBase r) {
if (running) {
node.onTimeout(r);
}
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#addDHTNode(java.lang.String, int)
*/
public void addDHTNode (String host, int hport) {
if (!running) {
return;
}
InetSocketAddress addr = new InetSocketAddress(host, hport);
if (!addr.isUnresolved()) {
if(!type.PREFERRED_ADDRESS_TYPE.isInstance(addr.getAddress()) || node.getNumEntriesInRoutingTable() > DHTConstants.BOOTSTRAP_IF_LESS_THAN_X_PEERS)
return;
srv.ping(node.getOurID(), addr);
}
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#announce(byte[], int)
*/
public AnnounceTask announce (byte[] info_hash, int port) {
if (!running) {
return null;
}
Key id = new Key(info_hash);
AnnounceTask at = new AnnounceTask(db, srv, node, id, port);
if (canStartTask()) {
at.start();
}
tman.addTask(at);
if (!db.contains(id)) {
db.insert(id);
}
return at;
}
public PingRefreshTask refreshBuckets (KBucket[] buckets,
boolean cleanOnTimeout) {
PingRefreshTask prt = new PingRefreshTask(srv, node, buckets,
cleanOnTimeout);
if (canStartTask()) {
prt.start();
}
tman.addTask(prt, true);
return prt;
}
public RPCServer getServer() {
return srv;
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#getClosestGoodNodes(int)
*/
public Map<String, Integer> getClosestGoodNodes (int maxNodes) {
Map<String, Integer> map = new HashMap<String, Integer>();
if (node == null) {
return map;
}
int max = 0;
KClosestNodesSearch kns = new KClosestNodesSearch(node.getOurID(), maxNodes * 2,this);
kns.fill();
for (KBucketEntry e : kns.getEntries()) {
if (!e.isGood()) {
continue;
}
InetSocketAddress a = e.getAddress();
map.put(a.getHostName(), a.getPort());
if (++max >= maxNodes) {
break;
}
}
return map;
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#getPort()
*/
public int getPort () {
return port;
}
public PopulationEstimator getEstimator() {
return estimator;
}
public DHTtype getType() {
return type;
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#getStats()
*/
public DHTStats getStats () {
return stats;
}
/**
* @return the status
*/
public DHTStatus getStatus () {
return status;
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#isRunning()
*/
public boolean isRunning () {
return running;
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#portRecieved(java.lang.String, int)
*/
public void portRecieved (String ip, int port) {
if (!running) {
return;
}
PingRequest r = new PingRequest(node.getOurID());
r.setOrigin(new InetSocketAddress(ip, port));
srv.doCall(r);
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#start(java.lang.String, int)
*/
public void start (File table, int port, boolean peerBootstrapOnly)
throws SocketException {
// if (running) {
// return;
// }
useRouterBootstrapping = !peerBootstrapOnly;
if (port == 0) {
port = 49001;
}
setStatus(DHTStatus.Initializing);
stats.resetStartedTimestamp();
table_file = table;
this.port = port;
System.out.println("Starting DHT on port " + port);
srv = new RPCServer(this, port);
stats.setRpcStats(srv.getStats());
node = new Node(srv);
db = new Database();
stats.setDbStats(db.getStats());
tman = new TaskManager();
expire_timer.update();
running = true;
scheduledActions.add(scheduler.scheduleAtFixedRate(new Runnable() {
public void run() {
// maintenance that should run all the time, before the first queries
tman.removeFinishedTasks(DHT.this);
if (running && hasStatsListeners()) {
onStatsUpdate();
}
}
}, 5000, DHTConstants.DHT_UPDATE_INTERVAL, TimeUnit.MILLISECONDS));
srv.start();
bootstrapping = true;
node.loadTable(table, this, new Runnable() {
public void run () {
started();
}
});
// // does 10k random lookups and prints them to a file for analysis
// scheduler.schedule(new Runnable() {
// //PrintWriter pw;
// TaskListener li = new TaskListener() {
// public synchronized void finished(Task t) {
// NodeLookup nl = ((NodeLookup) t);
// if (nl.closestSet.size() < DHTConstants.MAX_ENTRIES_PER_BUCKET)
// return;
// /*
// StringBuilder b = new StringBuilder();
// b.append(nl.targetKey.toString(false));
// b.append(",");
// for (Key i : nl.closestSet)
// b.append(i.toString(false).substring(0, 12) + ",");
// b.deleteCharAt(b.length() - 1);
// pw.println(b);
// pw.flush();
// */
// }
// };
//
// public void run() {
// if(type == DHTtype.IPV6_DHT)
// return;
// /*
// try
// {
// pw = new PrintWriter("H:\\mldht.log");
// } catch (FileNotFoundException e)
// {
// e.printStackTrace();
// }*/
// for (int i = 0; i < 10000; i++)
// {
// NodeLookup l = new NodeLookup(Key.createRandomKey(), srv, node, false);
// if (canStartTask())
// l.start();
// tman.addTask(l);
// l.addListener(li);
// if (i == (10000 - 1))
// l.addListener(new TaskListener() {
// public void finished(Task t) {
// System.out.println("10k lookups done");
// }
// });
// }
// }
// }, 1, TimeUnit.MINUTES);
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#started()
*/
public void started () {
bootstrapping = false;
bootstrap();
/*
if(type == DHTtype.IPV4_DHT)
{
Task t = new KeyspaceCrawler(srv, node);
tman.addTask(t);
}*/
scheduledActions.add(scheduler.scheduleAtFixedRate(new Runnable() {
public void run () {
try {
update();
} catch (RuntimeException e) {
log(e, LogLevel.Fatal);
}
}
}, 5000, DHTConstants.DHT_UPDATE_INTERVAL, TimeUnit.MILLISECONDS));
scheduledActions.add(scheduler.scheduleAtFixedRate(new Runnable() {
public void run () {
try {
findNode(Key.createRandomKey()).setInfo("Random Refresh Lookup");
} catch (RuntimeException e) {
log(e, LogLevel.Fatal);
}
}
}, DHTConstants.RANDOM_LOOKUP_INTERVAL, DHTConstants.RANDOM_LOOKUP_INTERVAL, TimeUnit.MILLISECONDS));
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#stop()
*/
public void stop () {
if (!running) {
return;
}
//scheduler.shutdown();
logInfo("Stopping DHT");
for (Task t : tman.getActiveTasks()) {
t.kill();
}
for(ScheduledFuture<?> future : scheduledActions)
future.cancel(false);
scheduler.getQueue().removeAll(scheduledActions);
scheduledActions.clear();
srv.stop();
try {
node.saveTable(table_file);
} catch (IOException e) {
e.printStackTrace();
}
running = false;
stopped();
tman = null;
db = null;
node = null;
srv = null;
setStatus(DHTStatus.Stopped);
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#getNode()
*/
public Node getNode () {
return node;
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#getTaskManager()
*/
public TaskManager getTaskManager () {
return tman;
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#stopped()
*/
public void stopped () {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#update()
*/
public void update () {
if (!running) {
return;
}
long now = System.currentTimeMillis();
if (expire_timer.getElapsedSinceUpdate() > DHTConstants.CHECK_FOR_EXPIRED_ENTRIES) {
db.expire(now);
expire_timer.update();
}
node.doBucketChecks(now);
if (!bootstrapping) {
if (node.getNumEntriesInRoutingTable() < DHTConstants.BOOTSTRAP_IF_LESS_THAN_X_PEERS) {
bootstrap();
} else if (now - lastBootstrap > DHTConstants.SELF_LOOKUP_INTERVAL) {
//update stale entries
PingRefreshTask prt = new PingRefreshTask(srv, node, false);
prt.setInfo("Refreshing old entries.");
if (canStartTask()) {
prt.start();
}
tman.addTask(prt, true);
//regualary search for our id to update routing table
bootstrap();
} else {
setStatus(DHTStatus.Running);
}
}
}
/**
* Initiates a Bootstrap.
*
* This function bootstraps with router.bittorrent.com if there are less
* than 10 Peers in the routing table. If there are more then a lookup on
* our own ID is initiated. If the either Task is finished than it will try
* to fill the Buckets.
*/
public synchronized void bootstrap () {
if (!running || bootstrapping || System.currentTimeMillis() - lastBootstrap < DHTConstants.BOOTSTRAP_MIN_INTERVAL) {
return;
}
List<InetSocketAddress> nodeAddresses = new ArrayList<InetSocketAddress>();
for(int i = 0;i<DHTConstants.BOOTSTRAP_NODES.length;i++)
{
try {
String hostname = DHTConstants.BOOTSTRAP_NODES[i];
int port = DHTConstants.BOOTSTRAP_PORTS[i];
for(InetAddress addr : InetAddress.getAllByName(hostname))
{
nodeAddresses.add(new InetSocketAddress(addr, port));
}
} catch (Exception e) {
// do nothing
}
}
if(nodeAddresses.size() > 0)
DHTConstants.BOOTSTRAP_NODE_ADDRESSES = nodeAddresses;
if (useRouterBootstrapping || node.getNumEntriesInRoutingTable() > 1) {
bootstrapping = true;
TaskListener bootstrapListener = new TaskListener() {
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.TaskListener#finished(lbms.plugins.mldht.kad.Task)
*/
public void finished (Task t) {
bootstrapping = false;
if (running
&& node.getNumEntriesInRoutingTable() > DHTConstants.USE_BT_ROUTER_IF_LESS_THAN_X_PEERS) {
node.fillBuckets(DHT.this);
}
}
};
logInfo("Bootstrapping...");
lastBootstrap = System.currentTimeMillis();
NodeLookup nl = findNode(node.getOurID(), true, true, true);
if (nl == null) {
bootstrapping = false;
} else if (node.getNumEntriesInRoutingTable() < DHTConstants.USE_BT_ROUTER_IF_LESS_THAN_X_PEERS) {
if (useRouterBootstrapping) {
List<InetSocketAddress> addrs = new ArrayList<InetSocketAddress>(DHTConstants.BOOTSTRAP_NODE_ADDRESSES);
Collections.shuffle(addrs);
for (InetSocketAddress addr : addrs)
{
if (!type.PREFERRED_ADDRESS_TYPE.isInstance(addr.getAddress()))
continue;
nl.addDHTNode(addr.getAddress(),addr.getPort());
break;
}
}
nl.addListener(bootstrapListener);
nl.setInfo("Bootstrap: Find Peers.");
tman.removeFinishedTasks(this);
} else {
nl.setInfo("Bootstrap: search for ourself.");
nl.addListener(bootstrapListener);
tman.removeFinishedTasks(this);
}
}
}
private NodeLookup findNode (Key id, boolean isBootstrap,
boolean isPriority, boolean queue) {
if (!running) {
return null;
}
NodeLookup at = new NodeLookup(id, srv, node, isBootstrap);
if (!queue && canStartTask()) {
at.start();
}
tman.addTask(at, isPriority);
return at;
}
/**
* Do a NodeLookup.
*
* @param id The id of the key to search
*/
public NodeLookup findNode (Key id) {
return findNode(id, false, false, true);
}
/*
* (non-Javadoc)
*
* @see lbms.plugins.mldht.kad.DHTBase#fillBucket(lbms.plugins.mldht.kad.KBucket)
*/
public NodeLookup fillBucket (Key id, KBucket bucket) {
bucket.updateRefreshTimer();
return findNode(id, false, true, true);
}
public PingRefreshTask refreshBucket (KBucket bucket) {
if (!running) {
return null;
}
PingRefreshTask prt = new PingRefreshTask(srv, node, bucket, false);
if (canStartTask()) {
prt.start();
}
tman.addTask(prt); // low priority, the bootstrap does a high prio one if necessary
return prt;
}
public void sendError (MessageBase origMsg, int code, String msg) {
sendError(origMsg.getOrigin(), origMsg.getMTID(), code, msg);
}
public void sendError (InetSocketAddress target, byte[] mtid, int code,
String msg) {
ErrorMessage errMsg = new ErrorMessage(mtid, code, msg);
errMsg.setDestination(target);
srv.sendMessage(errMsg);
}
public boolean canStartTask () {
// we can start a task if we have less then 7 runnning and
// there are at least 16 RPC slots available
if (tman.getNumTasks() >= DHTConstants.MAX_ACTIVE_TASKS) {
return false;
} else if (DHTConstants.MAX_ACTIVE_CALLS - srv.getNumActiveRPCCalls() <= 16) {
return false;
}
return true;
}
public Key getOurID () {
if (running) {
return node.getOurID();
}
return null;
}
private boolean hasStatsListeners () {
return !statsListeners.isEmpty();
}
private void onStatsUpdate () {
stats.setNumTasks(tman.getNumTasks() + tman.getNumQueuedTasks());
stats.setNumPeers(node.getNumEntriesInRoutingTable());
stats.setNumSentPackets(srv.getNumSent());
stats.setNumReceivedPackets(srv.getNumReceived());
stats.setNumRpcCalls(srv.getNumActiveRPCCalls());
for (int i = 0; i < statsListeners.size(); i++) {
statsListeners.get(i).statsUpdated(stats);
}
}
private void setStatus (DHTStatus status) {
if (!this.status.equals(status)) {
DHTStatus old = this.status;
this.status = status;
if (!statusListeners.isEmpty())
{
for (int i = 0; i < statusListeners.size(); i++)
{
statusListeners.get(i).statusChanged(status, old);
}
}
}
}
public void addStatsListener (DHTStatsListener listener) {
statsListeners.add(listener);
}
public void removeStatsListener (DHTStatsListener listener) {
statsListeners.remove(listener);
}
public void addStatusListener (DHTStatusListener listener) {
statusListeners.add(listener);
}
public void removeStatusListener (DHTStatusListener listener) {
statusListeners.remove(listener);
}
/**
* @return the logger
*/
// public static DHTLogger getLogger () {
// return logger;
// }
/**
* @param logger the logger to set
*/
public static void setLogger (DHTLogger logger) {
DHT.logger = logger;
}
/**
* @return the logLevel
*/
public static LogLevel getLogLevel () {
return logLevel;
}
/**
* @param logLevel the logLevel to set
*/
public static void setLogLevel (LogLevel logLevel) {
DHT.logLevel = logLevel;
logger.log("Change LogLevel to: " + logLevel);
}
/**
* @return the scheduler
*/
public static ScheduledExecutorService getScheduler () {
return scheduler;
}
public static void log (String message, LogLevel level) {
if (level.compareTo(logLevel) < 1) { // <=
logger.log(message);
}
}
public static void log (Exception e, LogLevel level) {
if (level.compareTo(logLevel) < 1) { // <=
logger.log(e);
}
}
public static void logFatal (String message) {
log(message, LogLevel.Fatal);
}
public static void logError (String message) {
log(message, LogLevel.Error);
}
public static void logInfo (String message) {
log(message, LogLevel.Info);
}
public static void logDebug (String message) {
log(message, LogLevel.Debug);
}
public static void logVerbose (String message) {
log(message, LogLevel.Verbose);
}
public static boolean isLogLevelEnabled (LogLevel level) {
return level.compareTo(logLevel) < 1;
}
public static enum LogLevel {
Fatal, Error, Info, Debug, Verbose
}
}