/**
* $Id: NettyRemotingAbstract.java 1831 2013-05-16 01:39:51Z shijia.wxr $
*/
package com.alibaba.rocketmq.remoting.netty;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.rocketmq.remoting.ChannelEventListener;
import com.alibaba.rocketmq.remoting.InvokeCallback;
import com.alibaba.rocketmq.remoting.common.Pair;
import com.alibaba.rocketmq.remoting.common.RemotingHelper;
import com.alibaba.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;
import com.alibaba.rocketmq.remoting.common.ServiceThread;
import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException;
import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException;
import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException;
import com.alibaba.rocketmq.remoting.protocol.RemotingCommand;
import com.alibaba.rocketmq.remoting.protocol.RemotingProtos.ResponseCode;
/**
* @author vintage.wang@gmail.com shijia.wxr@taobao.com
*
*/
public abstract class NettyRemotingAbstract {
private static final Logger plog = LoggerFactory.getLogger(RemotingHelper.RemotingLogName);
// �ź�����Oneway�����ʹ�ã���ֹ����Netty�����������
protected final Semaphore semaphoreOneway;
// �ź������첽���������ʹ�ã���ֹ����Netty�����������
protected final Semaphore semaphoreAsync;
// �������������
protected final ConcurrentHashMap<Integer /* opaque */, ResponseFuture> responseTable =
new ConcurrentHashMap<Integer, ResponseFuture>(256);
// Ĭ��������봦����
protected Pair<NettyRequestProcessor, Executor> defaultRequestProcessor;
// ע��ĸ���RPC������
protected final HashMap<Integer/* request code */, Pair<NettyRequestProcessor, Executor>> processorTable =
new HashMap<Integer, Pair<NettyRequestProcessor, Executor>>(64);
protected final NettyEventExecuter nettyEventExecuter = new NettyEventExecuter();
public abstract ChannelEventListener getChannelEventListener();
public void putNettyEvent(final NettyEvent event) {
this.nettyEventExecuter.putNettyEvent(event);
}
class NettyEventExecuter extends ServiceThread {
private final LinkedBlockingQueue<NettyEvent> eventQueue = new LinkedBlockingQueue<NettyEvent>();
private final int MaxSize = 10000;
public void putNettyEvent(final NettyEvent event) {
if (this.eventQueue.size() <= MaxSize) {
this.eventQueue.add(event);
}
else {
plog.warn("event queue size[{}] enough, so drop this event {}", this.eventQueue.size(),
event.toString());
}
}
@Override
public void run() {
plog.info(this.getServiceName() + " service started");
final ChannelEventListener listener = NettyRemotingAbstract.this.getChannelEventListener();
while (!this.isStoped()) {
try {
NettyEvent event = this.eventQueue.poll(3000, TimeUnit.MILLISECONDS);
if (event != null) {
switch (event.getType()) {
case CLOSE:
listener.onChannelClose(event.getRemoteAddr(), event.getChannel());
break;
case CONNECT:
listener.onChannelConnect(event.getRemoteAddr(), event.getChannel());
break;
case EXCEPTION:
listener.onChannelException(event.getRemoteAddr(), event.getChannel());
break;
default:
break;
}
}
}
catch (Exception e) {
plog.warn(this.getServiceName() + " service has exception. ", e);
}
}
plog.info(this.getServiceName() + " service end");
}
@Override
public String getServiceName() {
return NettyEventExecuter.class.getSimpleName();
}
}
public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) {
this.semaphoreOneway = new Semaphore(permitsOneway, true);
this.semaphoreAsync = new Semaphore(permitsAsync, true);
}
public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) {
final Pair<NettyRequestProcessor, Executor> matched = this.processorTable.get(cmd.getCode());
final Pair<NettyRequestProcessor, Executor> pair =
null == matched ? this.defaultRequestProcessor : matched;
if (pair != null) {
Runnable run = new Runnable() {
@Override
public void run() {
try {
final RemotingCommand response = pair.getObject1().processRequest(ctx, cmd);
if (cmd.isOnewayRPC()) {
if (response.getCode() != ResponseCode.SUCCESS_VALUE) {
plog.error("client oneway request has failed response, "
+ RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
plog.error(response.toString());
}
}
else {
if (response != null) {
response.setOpaque(cmd.getOpaque());
response.markResponseType();
try {
ctx.write(response).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
if (!future.isSuccess()) {
plog.error(
"response to "
+ RemotingHelper.parseChannelRemoteAddr(future
.channel()) + " failed", future.cause());
plog.error(cmd.toString());
plog.error(response.toString());
}
}
});
}
catch (Throwable e) {
plog.error("process request over, but response failed", e);
plog.error(cmd.toString());
plog.error(response.toString());
}
}
else {
// TODO
// �յ�������û�з���Ӧ�𣬿�����processRequest�н�����Ӧ�𣬺����������
}
}
}
catch (Throwable e) {
plog.error("process request exception", e);
plog.error(cmd.toString());
if (!cmd.isOnewayRPC()) {
final RemotingCommand response =
RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR_VALUE,
e.getMessage());
response.setOpaque(cmd.getOpaque());
ctx.write(response);
}
}
}
};
boolean executed = false;
for (int retry = 0; retry < 3 && !executed; retry++) {
try {
pair.getObject2().execute(run);
executed = true;
break;
}
catch (RejectedExecutionException e) {
plog.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel())
+ ", system thread pool busy, RejectedExecutionException "
+ pair.getObject2().toString());
try {
Thread.sleep(2);
}
catch (InterruptedException e1) {
plog.error("", e1);
}
}
}
if (!executed) {
if (!cmd.isOnewayRPC()) {
final RemotingCommand response =
RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_BUSY_VALUE,
"system busy now, please try another node");
response.setOpaque(cmd.getOpaque());
ctx.write(response);
}
}
}
else {
String error = " request type " + cmd.getCode() + " not supported";
final RemotingCommand response =
RemotingCommand.createResponseCommand(ResponseCode.REQUEST_CODE_NOT_SUPPORTED_VALUE, error);
response.setOpaque(cmd.getOpaque());
ctx.write(response);
plog.error(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + error);
}
}
public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) {
final ResponseFuture responseFuture = responseTable.get(cmd.getOpaque());
if (responseFuture != null) {
responseFuture.setResponseCommand(cmd);
responseFuture.release();
// �첽����
if (responseFuture.getInvokeCallback() != null) {
boolean runInThisThread = false;
Executor executor = this.getCallbackExecutor();
if (executor != null) {
try {
executor.execute(new Runnable() {
@Override
public void run() {
try {
responseFuture.getInvokeCallback().operationComplete(responseFuture);
}
catch (Throwable e) {
plog.warn("excute callback in executor exception, and callback throw", e);
}
}
});
}
catch (Exception e) {
runInThisThread = true;
plog.warn("excute callback in executor exception, maybe executor busy", e);
}
}
else {
runInThisThread = true;
}
if (runInThisThread) {
try {
responseFuture.getInvokeCallback().operationComplete(responseFuture);
}
catch (Throwable e) {
plog.warn("", e);
}
}
}
// ͬ������
else {
responseFuture.putResponse(cmd);
}
}
else {
plog.warn("receive response, but not matched any request, "
+ RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
plog.warn(cmd.toString());
}
responseTable.remove(cmd.getOpaque());
}
public void processMessageReceived(ChannelHandlerContext ctx, Object msg) throws Exception {
final RemotingCommand cmd = (RemotingCommand) msg;
if (cmd != null) {
switch (cmd.getType()) {
case REQUEST_COMMAND:
processRequestCommand(ctx, cmd);
break;
case RESPONSE_COMMAND:
processResponseCommand(ctx, cmd);
break;
default:
break;
}
}
}
abstract public Executor getCallbackExecutor();
public void scanResponseTable() {
List<Integer> todoList = new LinkedList<Integer>();
for (ResponseFuture rep : this.responseTable.values()) {
if ((rep.getBeginTimestamp() + rep.getTimeoutMillis() + 1000) <= System.currentTimeMillis()) {
todoList.add(rep.getOpaque());
if (rep.getInvokeCallback() != null) {
try {
rep.getInvokeCallback().operationComplete(rep);
}
catch (Throwable e) {
plog.error("scanResponseTable, operationComplete exception", e);
}
}
}
}
for (Integer opaque : todoList) {
ResponseFuture responseFuture = this.responseTable.remove(opaque);
if (responseFuture != null) {
responseFuture.release();
plog.warn("remove timeout request, " + responseFuture);
}
}
}
public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request,
final long timeoutMillis) throws InterruptedException, RemotingSendRequestException,
RemotingTimeoutException {
try {
final ResponseFuture responseFuture =
new ResponseFuture(request.getOpaque(), timeoutMillis, null, null);
this.responseTable.put(request.getOpaque(), responseFuture);
channel.write(request).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture f) throws Exception {
if (f.isSuccess()) {
responseFuture.setSendRequestOK(true);
return;
}
else {
responseFuture.setSendRequestOK(false);
}
responseTable.remove(request.getOpaque());
responseFuture.setCause(f.cause());
responseFuture.putResponse(null);
plog.warn("send a request command to channel <" + channel.remoteAddress() + "> failed.");
plog.warn(request.toString());
}
});
RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis);
if (null == responseCommand) {
// ��������ɹ�����ȡӦ��ʱ
if (responseFuture.isSendRequestOK()) {
throw new RemotingTimeoutException(RemotingHelper.parseChannelRemoteAddr(channel),
timeoutMillis, responseFuture.getCause());
}
// ��������ʧ��
else {
throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel),
responseFuture.getCause());
}
}
return responseCommand;
}
finally {
this.responseTable.remove(request.getOpaque());
}
}
public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis,
final InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException,
RemotingTimeoutException, RemotingSendRequestException {
boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
if (acquired) {
final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync);
final ResponseFuture responseFuture =
new ResponseFuture(request.getOpaque(), timeoutMillis, invokeCallback, once);
this.responseTable.put(request.getOpaque(), responseFuture);
try {
channel.write(request).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture f) throws Exception {
if (f.isSuccess()) {
responseFuture.setSendRequestOK(true);
return;
}
else {
responseFuture.setSendRequestOK(false);
}
once.release();
responseTable.remove(request.getOpaque());
responseFuture.putResponse(null);
plog.warn("send a request command to channel <" + channel.remoteAddress() + "> failed.");
plog.warn(request.toString());
}
});
}
catch (Exception e) {
once.release();
plog.warn("write send a request command to channel <" + channel.remoteAddress() + "> failed.");
throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);
}
}
else {
if (timeoutMillis <= 0) {
throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast");
}
else {
plog.warn("invokeAsyncImpl tryAcquire semaphore timeout, " + timeoutMillis
+ " waiting thread nums: " + this.semaphoreAsync.getQueueLength());
plog.warn(request.toString());
throw new RemotingTimeoutException("tryAcquire timeout(ms) " + timeoutMillis);
}
}
}
public void invokeOnewayImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis)
throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException,
RemotingSendRequestException {
request.markOnewayRPC();
boolean acquired = this.semaphoreOneway.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
if (acquired) {
final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreOneway);
try {
channel.write(request).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture f) throws Exception {
once.release();
if (!f.isSuccess()) {
plog.warn("send a request command to channel <" + channel.remoteAddress()
+ "> failed.");
plog.warn(request.toString());
}
}
});
}
catch (Exception e) {
once.release();
plog.warn("write send a request command to channel <" + channel.remoteAddress() + "> failed.");
throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);
}
}
else {
if (timeoutMillis <= 0) {
throw new RemotingTooMuchRequestException("invokeOnewayImpl invoke too fast");
}
else {
plog.warn("invokeOnewayImpl tryAcquire semaphore timeout, " + timeoutMillis
+ " waiting thread nums: " + this.semaphoreOneway.getQueueLength());
plog.warn(request.toString());
throw new RemotingTimeoutException("tryAcquire timeout(ms) " + timeoutMillis);
}
}
}
}