/**
* Copyright 2008 - CommonCrawl Foundation
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
**/
package org.commoncrawl.service.parser.client;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.commoncrawl.async.EventLoop;
import org.commoncrawl.async.Timer;
import org.commoncrawl.rpc.base.internal.AsyncClientChannel;
import org.commoncrawl.rpc.base.internal.AsyncRequest;
import org.commoncrawl.rpc.base.internal.NullMessage;
import org.commoncrawl.rpc.base.internal.AsyncClientChannel.ConnectionCallback;
import org.commoncrawl.rpc.base.internal.AsyncRequest.Callback;
import org.commoncrawl.rpc.base.internal.AsyncRequest.Status;
import org.commoncrawl.rpc.base.shared.RPCException;
import org.commoncrawl.service.parser.ParseRequest;
import org.commoncrawl.service.parser.ParseResult;
import org.commoncrawl.service.parser.ParserServiceSlave;
import org.commoncrawl.service.parser.SlaveStatus;
import org.commoncrawl.util.CCStringUtils;
/**
* A representation of an online parser node
*
* @author rana
*
*/
public class ParserNode implements ConnectionCallback, Comparable<ParserNode> {
String _nodeName;
InetSocketAddress _nodeAddress;
AsyncClientChannel _outgoingChannel;
ParserServiceSlave.AsyncStub _asyncStub;
AtomicBoolean _online = new AtomicBoolean();
EventLoop _eventLoop;
Dispatcher _dispatcher;
Timer _statusPollTimer;
SlaveStatus _status = null;
long _lastTouched = -1L;
public static final Log LOG = LogFactory.getLog(ParserNode.class);
private static final int TIMER_POLL_DELAY = 100;
public ParserNode(Dispatcher dispatcher,EventLoop eventLoop,String nodeName,InetSocketAddress address) {
_eventLoop = eventLoop;
_nodeName = nodeName;
_nodeAddress = address;
_dispatcher = dispatcher;
}
public String getNodeName() {
return _nodeName;
}
/**
* startup communications
*/
public void startup() throws IOException {
_outgoingChannel
= new AsyncClientChannel(
_eventLoop,
new InetSocketAddress(0),
_nodeAddress,this);
LOG.info("Starting Communications with Node:" + _nodeName);
_asyncStub = new ParserServiceSlave.AsyncStub(_outgoingChannel);
_outgoingChannel.open();
_statusPollTimer = new Timer(TIMER_POLL_DELAY,false,new Timer.Callback() {
@Override
public void timerFired(Timer timer) {
if (_online.get()) {
queryStatus();
}
else {
_dispatcher.getQueueLock().lock();
try {
_status = null;
}
finally {
_dispatcher.getQueueLock().unlock();
}
_dispatcher.nodeStatusChanged(ParserNode.this);
}
}
});
}
public void queryStatus(){
try {
// LOG.info("Querying Node:" + _nodeName + " for Status");
_asyncStub.queryStatus(new Callback<NullMessage, SlaveStatus>() {
@Override
public void requestComplete(AsyncRequest<NullMessage, SlaveStatus> request) {
_dispatcher.getQueueLock().lock();
try {
if (request.getStatus() == Status.Success) {
_status = request.getOutput();
LOG.info("Node:" + _nodeName +" Load:" + _status.getLoad()
+ " Queued:" + _status.getQueuedDocs()
+ " Active:" + _status.getActiveDocs());
}
else {
LOG.error("Failed to get Status from Node:" + _nodeName);
_status = null;
}
}
finally {
_dispatcher.getQueueLock().unlock();
}
_dispatcher.nodeStatusChanged(ParserNode.this);
_eventLoop.setTimer(_statusPollTimer);
}
});
} catch (RPCException e) {
// reset the timer ...
_eventLoop.setTimer(_statusPollTimer);
}
}
public ParseResult dispatchRequest(final ParseRequest request) throws IOException {
final AtomicReference<ParseResult> result = new AtomicReference<ParseResult>();
if (_online.get()) {
// LOG.info("Dispatching Parse Request for URL:" + request.getDocURL()
// + " to Node:" + _nodeName);
final Semaphore requestSemaphore = new Semaphore(0);
_eventLoop.queueAsyncCallback(new org.commoncrawl.async.Callback() {
@Override
public void execute() {
try {
_asyncStub.parseDocument(request, new Callback<ParseRequest, ParseResult>() {
@Override
public void requestComplete(
AsyncRequest<ParseRequest, ParseResult> request) {
try {
// LOG.info("Parse Request for URL:" + request.getInput().getDocURL()
// + " recvd responseStatus:" + request.getStatus()
// + " from Node:" + _nodeName);
if (request.getStatus() == Status.Success) {
result.set(request.getOutput());
}
}
finally {
// LOG.info("Releasing Request Semaphore for URL:" + request.getInput().getDocURL());
requestSemaphore.release();
}
}
});
} catch (Exception e) {
LOG.error(CCStringUtils.stringifyException(e));
LOG.info("Releasing Request Semaphore for URL:" + request.getDocURL());
requestSemaphore.release();
}
}
});
// LOG.info("Waiting on ParseReq Semaphore for URL:"+ request.getDocURL());
requestSemaphore.acquireUninterruptibly();
// LOG.info("ParseReq Semaphore signlaed for URL:"+ request.getDocURL());
}
return result.get();
}
public ParserServiceSlave.AsyncStub getStub() {
return _asyncStub;
}
/**
* shutdown communications
*/
public void shutdown() {
try {
_outgoingChannel.close();
} catch (IOException e) {
LOG.error(CCStringUtils.stringifyException(e));
}
_asyncStub = null;
}
@Override
public void OutgoingChannelConnected(AsyncClientChannel channel) {
_online.set(true);
queryStatus();
}
@Override
public boolean OutgoingChannelDisconnected(AsyncClientChannel channel) {
_eventLoop.cancelTimer(_statusPollTimer);
_online.set(false);
return false;
}
public boolean isOnline() {
return _online.get();
}
@Override
public int compareTo(ParserNode o) {
if (_status != null && o._status != null) {
int result = Double.compare(Math.floor(_status.getLoad()), Math.floor(o._status.getLoad()));
if (result == 0) {
result = (_status.getQueuedDocs() < o._status.getQueuedDocs()) ? -1 :
(_status.getQueuedDocs() > o._status.getQueuedDocs()) ? 1 : 0;
if (result == 0) {
result = (_status.getActiveDocs() < o._status.getActiveDocs()) ? -1 :
(_status.getActiveDocs() > o._status.getActiveDocs()) ? 1 : 0;
if (result == 0) {
result = (_lastTouched < o._lastTouched ) ? -1:
(_lastTouched > o._lastTouched ) ? 1 :0;
}
}
}
return result;
}
else if (_status != null && o._status == null) {
return -1;
}
else if (_status == null && o._status != null) {
}
else {
return (_lastTouched < o._lastTouched ) ? -1:
(_lastTouched > o._lastTouched ) ? 1 :0;
}
return 0;
}
public void touch() {
_dispatcher.getQueueLock().lock();
try {
// mark last touch time
_lastTouched = System.currentTimeMillis();
// invalidate status
_status = null;
}
finally {
_dispatcher.getQueueLock().unlock();
}
}
}