/*
* Copyright WSO2, Inc. (http://wso2.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.wso2.carbon.cloud.gateway.transport;
import org.apache.axiom.om.OMOutputFormat;
import org.apache.axiom.soap.SOAPEnvelope;
import org.apache.axis2.AxisFault;
import org.apache.axis2.Constants;
import org.apache.axis2.builder.BuilderUtil;
import org.apache.axis2.context.ConfigurationContext;
import org.apache.axis2.context.MessageContext;
import org.apache.axis2.description.Parameter;
import org.apache.axis2.description.TransportOutDescription;
import org.apache.axis2.engine.AxisEngine;
import org.apache.axis2.transport.OutTransportInfo;
import org.apache.axis2.transport.base.AbstractTransportSender;
import org.apache.axis2.transport.base.BaseUtils;
import org.apache.axis2.transport.base.threads.WorkerPool;
import org.apache.axis2.transport.base.threads.WorkerPoolFactory;
import org.apache.axis2.util.MessageContextBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.protocol.HTTP;
import org.apache.synapse.transport.nhttp.NhttpConstants;
import org.wso2.carbon.cloud.gateway.common.CGConstant;
import org.wso2.carbon.cloud.gateway.common.CGUtils;
import org.wso2.carbon.cloud.gateway.common.thrift.gen.Message;
import org.wso2.carbon.cloud.gateway.transport.server.CGThriftServerHandler;
import org.wso2.carbon.context.PrivilegedCarbonContext;
import org.wso2.carbon.relay.BinaryRelayBuilder;
import org.wso2.carbon.relay.ExpandingMessageFormatter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.*;
/**
* The CGTransport sender implementation. For one way messages this will just send the message to
* to the Thrift server's request message buffer using an in memory copy and for two way messages
* a semaphore will be blocked the current thread of execution until a response is received
*/
public class CGTransportSender extends AbstractTransportSender {
/**
* The time out for the semaphore
*/
private long semaphoreTimeOut;
/**
* The periodic task to clean up the dead messages in case of back end is gone
*/
private ScheduledExecutorService deadMsgCleanupScheduler;
/**
* The worker pool for processing
*/
private WorkerPool workerPool;
/**
* The builder for pass through
*/
private BinaryRelayBuilder builder;
/**
* The formatter for pass through
*/
private ExpandingMessageFormatter formatter;
private static Log log = LogFactory.getLog(CGTransportSender.class);
@Override
public void init(ConfigurationContext cfgCtx,
TransportOutDescription transportOut) throws AxisFault {
super.init(cfgCtx, transportOut);
builder = new BinaryRelayBuilder();
formatter = new ExpandingMessageFormatter();
semaphoreTimeOut = CGUtils.getLongProperty(CGConstant.CG_SEMAPHORE_TIMEOUT,
86400L);
int tenantId = PrivilegedCarbonContext.getThreadLocalCarbonContext().getTenantId();
String groupName = "CGTransportSender-tenant-" + tenantId + "-worker-thread-group";
String groupId = "CGTransportSender-tenant-" + tenantId + "-worker";
workerPool =
WorkerPoolFactory.getWorkerPool(
CGUtils.getIntProperty(
CGConstant.CG_T_CORE, CGConstant.WORKERS_CORE_THREADS),
CGUtils.getIntProperty(
CGConstant.CG_T_MAX, CGConstant.WORKERS_MAX_THREADS),
CGUtils.getIntProperty(
CGConstant.CG_T_ALIVE, CGConstant.WORKER_KEEP_ALIVE),
CGUtils.getIntProperty(
CGConstant.CG_T_QLEN, CGConstant.WORKER_BLOCKING_QUEUE_LENGTH),
groupName,
groupId);
//let the task run per once a day by default
String timeUnitAsString = CGUtils.getStringProperty(
CGConstant.TIME_UNIT, CGConstant.HOUR);
// both the scheduler and the idle message time will be used the same time unit
// given by CGConstant#TIME_UNIT
long noOfSchedulerTimeUnits = CGUtils.getLongProperty(
CGConstant.NO_OF_SCHEDULER_TIME_UNITS, 24L);
long noOfIdleMessageUnits = CGUtils.getLongProperty(
CGConstant.NO_OF_IDLE_MESSAGE_TIME_UNITS, 24L);
checkSchedulePreConditions(timeUnitAsString, noOfIdleMessageUnits, noOfSchedulerTimeUnits);
TimeUnit schedulerTimeUnit = getTimeUnit(timeUnitAsString);
// schedule the message clean up task in order to avoid server goes OOM in case of the
// back end server is offline
deadMsgCleanupScheduler = Executors.newSingleThreadScheduledExecutor();
deadMsgCleanupScheduler.scheduleWithFixedDelay(
new DeadMessageCleanupTask(
CGThriftServerHandler.getRequestBuffers(),
getDurationAsMillisecond(schedulerTimeUnit, noOfIdleMessageUnits)),
noOfSchedulerTimeUnits,
noOfSchedulerTimeUnits,
schedulerTimeUnit);
// start the response message dispatching tasks
int noOfDispatchingTask = CGUtils.getIntProperty(CGConstant.NO_OF_DISPATCH_TASK, 2);
for (int i = 0; i < noOfDispatchingTask; i++) {
workerPool.execute(new ResponseMessageDispatchingTask());
}
log.info("CGTransportSender started for tenant [" + tenantId + "]...");
}
@Override
public void cleanup(MessageContext msgContext) throws AxisFault {
super.cleanup(msgContext);
if (!deadMsgCleanupScheduler.isShutdown()) {
deadMsgCleanupScheduler.shutdown();
}
}
@Override
public void stop() {
super.stop();
}
@Override
public void sendMessage(MessageContext msgContext,
String targetEPR,
OutTransportInfo outTransportInfo) throws AxisFault {
try {
String requestUri = (String) msgContext.getProperty(
Constants.Configuration.TRANSPORT_IN_URL);
if (requestUri == null) {
handleException("The request URI is null");
}
String endpointPrefix = (String)msgContext.getProperty(NhttpConstants.ENDPOINT_PREFIX);
if (endpointPrefix == null) {
handleException("The ENDPOINT_PREFIX(EPR) is not found");
}
Object headers = msgContext.getProperty(
org.apache.axis2.context.MessageContext.TRANSPORT_HEADERS);
if (headers == null) {
handleException("Transport headers are null");
}
String requestMsgIdMsgId = msgContext.getMessageID();
if (requestMsgIdMsgId == null) {
requestMsgIdMsgId = UUID.randomUUID().toString();
}
Message thriftMsg = new Message();
if (msgContext.isDoingMTOM()) {
thriftMsg.setIsDoingMTOM(msgContext.isDoingMTOM());
msgContext.setProperty(
org.apache.axis2.Constants.Configuration.ENABLE_MTOM,
org.apache.axis2.Constants.VALUE_TRUE);
} else if (msgContext.isDoingSwA()) {
thriftMsg.setIsDoingSwA(msgContext.isDoingSwA());
msgContext.setProperty(
org.apache.axis2.Constants.Configuration.ENABLE_SWA,
org.apache.axis2.Constants.VALUE_TRUE);
} else if (msgContext.isDoingREST()) {
thriftMsg.setIsDoingREST(msgContext.isDoingREST());
}
thriftMsg.setHttpMethod((String) msgContext.getProperty(
Constants.Configuration.HTTP_METHOD));
thriftMsg.setMessageId(requestMsgIdMsgId);
thriftMsg.setEpoch(System.currentTimeMillis());
// a class cast exception (if any) will be logged in case mismatch type is returned,
// we will not worry about the type because correct type should be returned
thriftMsg.setRequestURI(requestUri);
thriftMsg.setSoapAction(msgContext.getSoapAction());
OMOutputFormat format = BaseUtils.getOMOutputFormat(msgContext);
ByteArrayOutputStream out = new ByteArrayOutputStream();
formatter.writeTo(msgContext, format, out, false);
thriftMsg.setMessage(out.toByteArray());
String contentType = formatter.getContentType(msgContext, format, msgContext.getSoapAction());
thriftMsg.setContentType(contentType);
if (((Map) headers).containsKey(HTTP.CONTENT_TYPE)) {
((Map) headers).put(HTTP.CONTENT_TYPE, contentType);
}
thriftMsg.setTransportHeaders((Map) headers);
Semaphore available = null;
// The csg polling transport on the other side will directly use the EPR as the key for
// message buffer. Although this introduce a tight couple between the CGTransport
// and CGPollingTransport this is done this way to achieve maximum performance
String token = CGThriftServerHandler.getSecureUUID(endpointPrefix);
if (token == null) {
handleException("No permission to access the server buffers");
}
boolean isOutIn = waitForSynchronousResponse(msgContext);
if (isOutIn) {
available = new Semaphore(0, true);
CGThriftServerHandler.getSemaphoreMap().put(requestMsgIdMsgId, available);
}
CGThriftServerHandler.addRequestMessage(thriftMsg, token);
try {
if (isOutIn) {
// wait until the response is available, this thread will signal by the
// semaphore checking thread or send a timeout error if there is no response
// with the configured semaphore timeout or if the semaphore received an
// interrupted exception
try {
available.tryAcquire(semaphoreTimeOut, TimeUnit.SECONDS);
} catch (InterruptedException ignore) {
}
// make sure we don't run out of the main memory
CGThriftServerHandler.getSemaphoreMap().remove(requestMsgIdMsgId);
Message msg = CGThriftServerHandler.getMiddleBuffer().remove(requestMsgIdMsgId);
if (msg != null) {
handleSyncResponse(msgContext, msg, contentType);
} else {
// we don't have a response come yet, so send a fault to client
log.warn("The semaphore with id '" + requestMsgIdMsgId + "' was time out while "
+ "waiting for a response, sending a fault to client..");
sendFault(msgContext,
new Exception("Times out occurs while waiting for a response"));
}
}
} catch (Exception e) {
handleException("Could not process the response message", e);
}
} catch (Exception e) {
handleException("Could not process the request message", e);
}
}
private void handleSyncResponse(MessageContext requestMsgCtx, Message message, String requestContentType)
throws AxisFault {
try {
MessageContext responseMsgCtx = createResponseMessageContext(requestMsgCtx);
// set the message type of the original message, this is required for REST to work
// properly
responseMsgCtx.setProperty(Constants.Configuration.MESSAGE_TYPE,
requestMsgCtx.getProperty(Constants.Configuration.MESSAGE_TYPE));
responseMsgCtx.setProperty(Constants.Configuration.CONTENT_TYPE,
requestMsgCtx.getProperty(Constants.Configuration.CONTENT_TYPE));
if (message.isIsDoingMTOM()) {
responseMsgCtx.setProperty(
org.apache.axis2.Constants.Configuration.ENABLE_MTOM,
org.apache.axis2.Constants.VALUE_TRUE);
} else if (message.isIsDoingSwA()) {
responseMsgCtx.setProperty(
org.apache.axis2.Constants.Configuration.ENABLE_SWA,
org.apache.axis2.Constants.VALUE_TRUE);
}
String contentType = message.getContentType();
if (contentType == null) {
contentType = inferContentType(requestContentType, responseMsgCtx);
}
ByteArrayInputStream inputStream = new ByteArrayInputStream(message.getMessage());
// a class cast will be thrown if incorrect type was return, we are not worrying about
// that because that should be handle by the builder
SOAPEnvelope envelope = (SOAPEnvelope) builder.processDocument(
inputStream, contentType, responseMsgCtx);
responseMsgCtx.setEnvelope(envelope);
String charSetEnc = BuilderUtil.getCharSetEncoding(contentType);
if (charSetEnc == null) {
charSetEnc = MessageContext.DEFAULT_CHAR_SET_ENCODING;
}
responseMsgCtx.setProperty(
Constants.Configuration.CHARACTER_SET_ENCODING,
contentType.indexOf(HTTP.CHARSET_PARAM) > 0
? charSetEnc : MessageContext.DEFAULT_CHAR_SET_ENCODING);
responseMsgCtx.setProperty(
MessageContext.TRANSPORT_HEADERS, message.getTransportHeaders());
if (message.getSoapAction() != null) {
responseMsgCtx.setSoapAction(message.getSoapAction());
}
AxisEngine.receive(responseMsgCtx);
} catch (AxisFault axisFault) {
handleException("Could not handle the response message ", axisFault);
}
}
private void sendFault(MessageContext msgContext, Exception e) {
try {
MessageContext faultContext = MessageContextBuilder.createFaultMessageContext(
msgContext, e);
faultContext.setProperty("ERROR_MESSAGE", e.getMessage());
faultContext.setProperty("SENDING_FAULT", Boolean.TRUE);
AxisEngine.sendFault(faultContext);
} catch (AxisFault axisFault) {
log.fatal("Could not create the fault message.", axisFault);
}
}
/**
* A periodic task which submit the response for processing
*/
private class ResponseMessageDispatchingTask implements Runnable {
public void run() {
while (true) {
// if there is no response messages the current thread will block,
// BlockingQueue#drainTo drains a block of message but it doesn't seems block
// without eating up the CPU
Message msg = CGThriftServerHandler.getResponseMessage();
if (msg != null) {
workerPool.execute(new ResponseMessageProcessingTask(msg));
}
}
}
}
/**
* The task which send the response message back to client
*/
private class ResponseMessageProcessingTask implements Runnable {
private Message msg;
private ResponseMessageProcessingTask(Message msg) {
this.msg = msg;
}
public void run() {
String msgId = msg.getMessageId();
Map<String, Semaphore> semaphoreMap = CGThriftServerHandler.getSemaphoreMap();
Set<String> keySet = semaphoreMap.keySet();
if (keySet.contains(msgId)) {
CGThriftServerHandler.getMiddleBuffer().put(msgId, msg);
Semaphore semaphore = semaphoreMap.get(msgId);
semaphore.release();
} else {
log.warn("A response was received with id '" + msgId + "', but no registered" +
" call back found. Message will be ignored!");
}
}
}
/**
* A cleanup task to remove the messages from the server buffers in case the back end has gone
*/
private class DeadMessageCleanupTask implements Runnable {
private Map<String, BlockingQueue<Message>> requestMessageBuffers;
private long idleMessageTime;
private DeadMessageCleanupTask(Map<String, BlockingQueue<Message>> requestMessageBuffers,
long idleMessageTime) {
this.requestMessageBuffers = requestMessageBuffers;
this.idleMessageTime = idleMessageTime;
}
public void run() {
long currentTime = System.currentTimeMillis();
for (Map.Entry<String, BlockingQueue<Message>> entry : requestMessageBuffers.entrySet()) {
BlockingQueue<Message> buffer = entry.getValue();
Message msg = buffer.peek();
while (msg != null && (msg.getEpoch() + idleMessageTime) > currentTime) {
String msgID = msg.getMessageId();
log.info("The cleaning up task is sweeping the message with id '"
+ msgID + "' and callback will be removed too.");
CGThriftServerHandler.getSemaphoreMap().remove(msgID);
buffer.remove();
msg = buffer.peek();
}
}
}
}
private static TimeUnit getTimeUnit(String timeUnit) {
if (timeUnit.equals(CGConstant.MILLISECOND)) {
return TimeUnit.MILLISECONDS;
} else if (timeUnit.equals(CGConstant.SECOND)) {
return TimeUnit.SECONDS;
} else if (timeUnit.equals(CGConstant.MINUTE)) {
return TimeUnit.MINUTES;
} else if (timeUnit.equals(CGConstant.HOUR)) {
return TimeUnit.HOURS;
} else if (timeUnit.equals(CGConstant.DAY)) {
return TimeUnit.DAYS;
} else {
// the default
return TimeUnit.DAYS;
}
}
private static void checkSchedulePreConditions(String timeUnits,
long noOfIdleMsgTimeUnits,
long noOfSchedulerTimeUnits) throws AxisFault {
if (noOfIdleMsgTimeUnits > noOfSchedulerTimeUnits) {
String msg = "A possible configuration error. The ScheduledExecutorService is " +
"configured to run once a every '" + noOfSchedulerTimeUnits + "' " +
(noOfSchedulerTimeUnits == 1 ? timeUnits : timeUnits + "s") + " to sweep " +
"messages which are '" + noOfIdleMsgTimeUnits + "' " +
(noOfIdleMsgTimeUnits == 1 ? timeUnits : timeUnits + "s") + "old. The " +
"scheduler may idle without doing any actual work!";
log.error(msg);
throw new AxisFault(msg);
}
}
private static long getDurationAsMillisecond(TimeUnit timeUnit, long duration) {
if (timeUnit == TimeUnit.MILLISECONDS) {
return TimeUnit.MILLISECONDS.toMillis(duration);
} else if (timeUnit == TimeUnit.SECONDS) {
return TimeUnit.SECONDS.toMillis(duration);
} else if (timeUnit == TimeUnit.MINUTES) {
return TimeUnit.MINUTES.toMillis(duration);
} else if (timeUnit == TimeUnit.HOURS) {
return TimeUnit.HOURS.toMillis(duration);
} else if (timeUnit == TimeUnit.DAYS) {
return TimeUnit.DAYS.toMillis(duration);
} else {
log.warn("TimeUnit type '" + timeUnit + "' is not supported. Default TimeUnit will be " +
"assumed");
return TimeUnit.DAYS.toMillis(duration);
}
}
private String inferContentType(String requestContentType, MessageContext responseMsgCtx) {
// Try to get the content type from the message context
Object cTypeProperty = responseMsgCtx.getProperty(Constants.Configuration.CONTENT_TYPE);
if (cTypeProperty != null) {
return cTypeProperty.toString();
}
// Try to get the content type from the axis configuration
Parameter cTypeParam = cfgCtx.getAxisConfiguration().getParameter(
Constants.Configuration.CONTENT_TYPE);
if (cTypeParam != null) {
return cTypeParam.getValue().toString();
}
if (requestContentType != null) {
return requestContentType;
}
// Unable to determine the content type - Return default value
return CGConstant.DEFAULT_CONTENT_TYPE;
}
private static String calculateBufferKey(String fullEPR) {
// cg://server1/SimpleStockQuoteService/operation1/argument1
// 5 is the length(cg://) used this way for better performance
String split[] = fullEPR.substring(5).split("/");
StringBuilder buf = new StringBuilder(CGConstant.CG_TRANSPORT_PREFIX);
// following is not thread safe, but there will be a thread per request
// http://vanillajava.blogspot.com/2012/08/java-memes-which-refuse-to-die.html
buf.append(split[0]).append("/").append(split[1]);
return buf.toString();
}
}