/* * Copyright WSO2, Inc. (http://wso2.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.wso2.carbon.cloud.gateway.transport; import org.apache.axiom.om.OMOutputFormat; import org.apache.axiom.soap.SOAPEnvelope; import org.apache.axis2.AxisFault; import org.apache.axis2.Constants; import org.apache.axis2.builder.BuilderUtil; import org.apache.axis2.context.ConfigurationContext; import org.apache.axis2.context.MessageContext; import org.apache.axis2.description.Parameter; import org.apache.axis2.description.TransportOutDescription; import org.apache.axis2.engine.AxisEngine; import org.apache.axis2.transport.OutTransportInfo; import org.apache.axis2.transport.base.AbstractTransportSender; import org.apache.axis2.transport.base.BaseUtils; import org.apache.axis2.transport.base.threads.WorkerPool; import org.apache.axis2.transport.base.threads.WorkerPoolFactory; import org.apache.axis2.util.MessageContextBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.protocol.HTTP; import org.apache.synapse.transport.nhttp.NhttpConstants; import org.wso2.carbon.cloud.gateway.common.CGConstant; import org.wso2.carbon.cloud.gateway.common.CGUtils; import org.wso2.carbon.cloud.gateway.common.thrift.gen.Message; import org.wso2.carbon.cloud.gateway.transport.server.CGThriftServerHandler; import org.wso2.carbon.context.PrivilegedCarbonContext; import org.wso2.carbon.relay.BinaryRelayBuilder; import org.wso2.carbon.relay.ExpandingMessageFormatter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.*; /** * The CGTransport sender implementation. For one way messages this will just send the message to * to the Thrift server's request message buffer using an in memory copy and for two way messages * a semaphore will be blocked the current thread of execution until a response is received */ public class CGTransportSender extends AbstractTransportSender { /** * The time out for the semaphore */ private long semaphoreTimeOut; /** * The periodic task to clean up the dead messages in case of back end is gone */ private ScheduledExecutorService deadMsgCleanupScheduler; /** * The worker pool for processing */ private WorkerPool workerPool; /** * The builder for pass through */ private BinaryRelayBuilder builder; /** * The formatter for pass through */ private ExpandingMessageFormatter formatter; private static Log log = LogFactory.getLog(CGTransportSender.class); @Override public void init(ConfigurationContext cfgCtx, TransportOutDescription transportOut) throws AxisFault { super.init(cfgCtx, transportOut); builder = new BinaryRelayBuilder(); formatter = new ExpandingMessageFormatter(); semaphoreTimeOut = CGUtils.getLongProperty(CGConstant.CG_SEMAPHORE_TIMEOUT, 86400L); int tenantId = PrivilegedCarbonContext.getThreadLocalCarbonContext().getTenantId(); String groupName = "CGTransportSender-tenant-" + tenantId + "-worker-thread-group"; String groupId = "CGTransportSender-tenant-" + tenantId + "-worker"; workerPool = WorkerPoolFactory.getWorkerPool( CGUtils.getIntProperty( CGConstant.CG_T_CORE, CGConstant.WORKERS_CORE_THREADS), CGUtils.getIntProperty( CGConstant.CG_T_MAX, CGConstant.WORKERS_MAX_THREADS), CGUtils.getIntProperty( CGConstant.CG_T_ALIVE, CGConstant.WORKER_KEEP_ALIVE), CGUtils.getIntProperty( CGConstant.CG_T_QLEN, CGConstant.WORKER_BLOCKING_QUEUE_LENGTH), groupName, groupId); //let the task run per once a day by default String timeUnitAsString = CGUtils.getStringProperty( CGConstant.TIME_UNIT, CGConstant.HOUR); // both the scheduler and the idle message time will be used the same time unit // given by CGConstant#TIME_UNIT long noOfSchedulerTimeUnits = CGUtils.getLongProperty( CGConstant.NO_OF_SCHEDULER_TIME_UNITS, 24L); long noOfIdleMessageUnits = CGUtils.getLongProperty( CGConstant.NO_OF_IDLE_MESSAGE_TIME_UNITS, 24L); checkSchedulePreConditions(timeUnitAsString, noOfIdleMessageUnits, noOfSchedulerTimeUnits); TimeUnit schedulerTimeUnit = getTimeUnit(timeUnitAsString); // schedule the message clean up task in order to avoid server goes OOM in case of the // back end server is offline deadMsgCleanupScheduler = Executors.newSingleThreadScheduledExecutor(); deadMsgCleanupScheduler.scheduleWithFixedDelay( new DeadMessageCleanupTask( CGThriftServerHandler.getRequestBuffers(), getDurationAsMillisecond(schedulerTimeUnit, noOfIdleMessageUnits)), noOfSchedulerTimeUnits, noOfSchedulerTimeUnits, schedulerTimeUnit); // start the response message dispatching tasks int noOfDispatchingTask = CGUtils.getIntProperty(CGConstant.NO_OF_DISPATCH_TASK, 2); for (int i = 0; i < noOfDispatchingTask; i++) { workerPool.execute(new ResponseMessageDispatchingTask()); } log.info("CGTransportSender started for tenant [" + tenantId + "]..."); } @Override public void cleanup(MessageContext msgContext) throws AxisFault { super.cleanup(msgContext); if (!deadMsgCleanupScheduler.isShutdown()) { deadMsgCleanupScheduler.shutdown(); } } @Override public void stop() { super.stop(); } @Override public void sendMessage(MessageContext msgContext, String targetEPR, OutTransportInfo outTransportInfo) throws AxisFault { try { String requestUri = (String) msgContext.getProperty( Constants.Configuration.TRANSPORT_IN_URL); if (requestUri == null) { handleException("The request URI is null"); } String endpointPrefix = (String)msgContext.getProperty(NhttpConstants.ENDPOINT_PREFIX); if (endpointPrefix == null) { handleException("The ENDPOINT_PREFIX(EPR) is not found"); } Object headers = msgContext.getProperty( org.apache.axis2.context.MessageContext.TRANSPORT_HEADERS); if (headers == null) { handleException("Transport headers are null"); } String requestMsgIdMsgId = msgContext.getMessageID(); if (requestMsgIdMsgId == null) { requestMsgIdMsgId = UUID.randomUUID().toString(); } Message thriftMsg = new Message(); if (msgContext.isDoingMTOM()) { thriftMsg.setIsDoingMTOM(msgContext.isDoingMTOM()); msgContext.setProperty( org.apache.axis2.Constants.Configuration.ENABLE_MTOM, org.apache.axis2.Constants.VALUE_TRUE); } else if (msgContext.isDoingSwA()) { thriftMsg.setIsDoingSwA(msgContext.isDoingSwA()); msgContext.setProperty( org.apache.axis2.Constants.Configuration.ENABLE_SWA, org.apache.axis2.Constants.VALUE_TRUE); } else if (msgContext.isDoingREST()) { thriftMsg.setIsDoingREST(msgContext.isDoingREST()); } thriftMsg.setHttpMethod((String) msgContext.getProperty( Constants.Configuration.HTTP_METHOD)); thriftMsg.setMessageId(requestMsgIdMsgId); thriftMsg.setEpoch(System.currentTimeMillis()); // a class cast exception (if any) will be logged in case mismatch type is returned, // we will not worry about the type because correct type should be returned thriftMsg.setRequestURI(requestUri); thriftMsg.setSoapAction(msgContext.getSoapAction()); OMOutputFormat format = BaseUtils.getOMOutputFormat(msgContext); ByteArrayOutputStream out = new ByteArrayOutputStream(); formatter.writeTo(msgContext, format, out, false); thriftMsg.setMessage(out.toByteArray()); String contentType = formatter.getContentType(msgContext, format, msgContext.getSoapAction()); thriftMsg.setContentType(contentType); if (((Map) headers).containsKey(HTTP.CONTENT_TYPE)) { ((Map) headers).put(HTTP.CONTENT_TYPE, contentType); } thriftMsg.setTransportHeaders((Map) headers); Semaphore available = null; // The csg polling transport on the other side will directly use the EPR as the key for // message buffer. Although this introduce a tight couple between the CGTransport // and CGPollingTransport this is done this way to achieve maximum performance String token = CGThriftServerHandler.getSecureUUID(endpointPrefix); if (token == null) { handleException("No permission to access the server buffers"); } boolean isOutIn = waitForSynchronousResponse(msgContext); if (isOutIn) { available = new Semaphore(0, true); CGThriftServerHandler.getSemaphoreMap().put(requestMsgIdMsgId, available); } CGThriftServerHandler.addRequestMessage(thriftMsg, token); try { if (isOutIn) { // wait until the response is available, this thread will signal by the // semaphore checking thread or send a timeout error if there is no response // with the configured semaphore timeout or if the semaphore received an // interrupted exception try { available.tryAcquire(semaphoreTimeOut, TimeUnit.SECONDS); } catch (InterruptedException ignore) { } // make sure we don't run out of the main memory CGThriftServerHandler.getSemaphoreMap().remove(requestMsgIdMsgId); Message msg = CGThriftServerHandler.getMiddleBuffer().remove(requestMsgIdMsgId); if (msg != null) { handleSyncResponse(msgContext, msg, contentType); } else { // we don't have a response come yet, so send a fault to client log.warn("The semaphore with id '" + requestMsgIdMsgId + "' was time out while " + "waiting for a response, sending a fault to client.."); sendFault(msgContext, new Exception("Times out occurs while waiting for a response")); } } } catch (Exception e) { handleException("Could not process the response message", e); } } catch (Exception e) { handleException("Could not process the request message", e); } } private void handleSyncResponse(MessageContext requestMsgCtx, Message message, String requestContentType) throws AxisFault { try { MessageContext responseMsgCtx = createResponseMessageContext(requestMsgCtx); // set the message type of the original message, this is required for REST to work // properly responseMsgCtx.setProperty(Constants.Configuration.MESSAGE_TYPE, requestMsgCtx.getProperty(Constants.Configuration.MESSAGE_TYPE)); responseMsgCtx.setProperty(Constants.Configuration.CONTENT_TYPE, requestMsgCtx.getProperty(Constants.Configuration.CONTENT_TYPE)); if (message.isIsDoingMTOM()) { responseMsgCtx.setProperty( org.apache.axis2.Constants.Configuration.ENABLE_MTOM, org.apache.axis2.Constants.VALUE_TRUE); } else if (message.isIsDoingSwA()) { responseMsgCtx.setProperty( org.apache.axis2.Constants.Configuration.ENABLE_SWA, org.apache.axis2.Constants.VALUE_TRUE); } String contentType = message.getContentType(); if (contentType == null) { contentType = inferContentType(requestContentType, responseMsgCtx); } ByteArrayInputStream inputStream = new ByteArrayInputStream(message.getMessage()); // a class cast will be thrown if incorrect type was return, we are not worrying about // that because that should be handle by the builder SOAPEnvelope envelope = (SOAPEnvelope) builder.processDocument( inputStream, contentType, responseMsgCtx); responseMsgCtx.setEnvelope(envelope); String charSetEnc = BuilderUtil.getCharSetEncoding(contentType); if (charSetEnc == null) { charSetEnc = MessageContext.DEFAULT_CHAR_SET_ENCODING; } responseMsgCtx.setProperty( Constants.Configuration.CHARACTER_SET_ENCODING, contentType.indexOf(HTTP.CHARSET_PARAM) > 0 ? charSetEnc : MessageContext.DEFAULT_CHAR_SET_ENCODING); responseMsgCtx.setProperty( MessageContext.TRANSPORT_HEADERS, message.getTransportHeaders()); if (message.getSoapAction() != null) { responseMsgCtx.setSoapAction(message.getSoapAction()); } AxisEngine.receive(responseMsgCtx); } catch (AxisFault axisFault) { handleException("Could not handle the response message ", axisFault); } } private void sendFault(MessageContext msgContext, Exception e) { try { MessageContext faultContext = MessageContextBuilder.createFaultMessageContext( msgContext, e); faultContext.setProperty("ERROR_MESSAGE", e.getMessage()); faultContext.setProperty("SENDING_FAULT", Boolean.TRUE); AxisEngine.sendFault(faultContext); } catch (AxisFault axisFault) { log.fatal("Could not create the fault message.", axisFault); } } /** * A periodic task which submit the response for processing */ private class ResponseMessageDispatchingTask implements Runnable { public void run() { while (true) { // if there is no response messages the current thread will block, // BlockingQueue#drainTo drains a block of message but it doesn't seems block // without eating up the CPU Message msg = CGThriftServerHandler.getResponseMessage(); if (msg != null) { workerPool.execute(new ResponseMessageProcessingTask(msg)); } } } } /** * The task which send the response message back to client */ private class ResponseMessageProcessingTask implements Runnable { private Message msg; private ResponseMessageProcessingTask(Message msg) { this.msg = msg; } public void run() { String msgId = msg.getMessageId(); Map<String, Semaphore> semaphoreMap = CGThriftServerHandler.getSemaphoreMap(); Set<String> keySet = semaphoreMap.keySet(); if (keySet.contains(msgId)) { CGThriftServerHandler.getMiddleBuffer().put(msgId, msg); Semaphore semaphore = semaphoreMap.get(msgId); semaphore.release(); } else { log.warn("A response was received with id '" + msgId + "', but no registered" + " call back found. Message will be ignored!"); } } } /** * A cleanup task to remove the messages from the server buffers in case the back end has gone */ private class DeadMessageCleanupTask implements Runnable { private Map<String, BlockingQueue<Message>> requestMessageBuffers; private long idleMessageTime; private DeadMessageCleanupTask(Map<String, BlockingQueue<Message>> requestMessageBuffers, long idleMessageTime) { this.requestMessageBuffers = requestMessageBuffers; this.idleMessageTime = idleMessageTime; } public void run() { long currentTime = System.currentTimeMillis(); for (Map.Entry<String, BlockingQueue<Message>> entry : requestMessageBuffers.entrySet()) { BlockingQueue<Message> buffer = entry.getValue(); Message msg = buffer.peek(); while (msg != null && (msg.getEpoch() + idleMessageTime) > currentTime) { String msgID = msg.getMessageId(); log.info("The cleaning up task is sweeping the message with id '" + msgID + "' and callback will be removed too."); CGThriftServerHandler.getSemaphoreMap().remove(msgID); buffer.remove(); msg = buffer.peek(); } } } } private static TimeUnit getTimeUnit(String timeUnit) { if (timeUnit.equals(CGConstant.MILLISECOND)) { return TimeUnit.MILLISECONDS; } else if (timeUnit.equals(CGConstant.SECOND)) { return TimeUnit.SECONDS; } else if (timeUnit.equals(CGConstant.MINUTE)) { return TimeUnit.MINUTES; } else if (timeUnit.equals(CGConstant.HOUR)) { return TimeUnit.HOURS; } else if (timeUnit.equals(CGConstant.DAY)) { return TimeUnit.DAYS; } else { // the default return TimeUnit.DAYS; } } private static void checkSchedulePreConditions(String timeUnits, long noOfIdleMsgTimeUnits, long noOfSchedulerTimeUnits) throws AxisFault { if (noOfIdleMsgTimeUnits > noOfSchedulerTimeUnits) { String msg = "A possible configuration error. The ScheduledExecutorService is " + "configured to run once a every '" + noOfSchedulerTimeUnits + "' " + (noOfSchedulerTimeUnits == 1 ? timeUnits : timeUnits + "s") + " to sweep " + "messages which are '" + noOfIdleMsgTimeUnits + "' " + (noOfIdleMsgTimeUnits == 1 ? timeUnits : timeUnits + "s") + "old. The " + "scheduler may idle without doing any actual work!"; log.error(msg); throw new AxisFault(msg); } } private static long getDurationAsMillisecond(TimeUnit timeUnit, long duration) { if (timeUnit == TimeUnit.MILLISECONDS) { return TimeUnit.MILLISECONDS.toMillis(duration); } else if (timeUnit == TimeUnit.SECONDS) { return TimeUnit.SECONDS.toMillis(duration); } else if (timeUnit == TimeUnit.MINUTES) { return TimeUnit.MINUTES.toMillis(duration); } else if (timeUnit == TimeUnit.HOURS) { return TimeUnit.HOURS.toMillis(duration); } else if (timeUnit == TimeUnit.DAYS) { return TimeUnit.DAYS.toMillis(duration); } else { log.warn("TimeUnit type '" + timeUnit + "' is not supported. Default TimeUnit will be " + "assumed"); return TimeUnit.DAYS.toMillis(duration); } } private String inferContentType(String requestContentType, MessageContext responseMsgCtx) { // Try to get the content type from the message context Object cTypeProperty = responseMsgCtx.getProperty(Constants.Configuration.CONTENT_TYPE); if (cTypeProperty != null) { return cTypeProperty.toString(); } // Try to get the content type from the axis configuration Parameter cTypeParam = cfgCtx.getAxisConfiguration().getParameter( Constants.Configuration.CONTENT_TYPE); if (cTypeParam != null) { return cTypeParam.getValue().toString(); } if (requestContentType != null) { return requestContentType; } // Unable to determine the content type - Return default value return CGConstant.DEFAULT_CONTENT_TYPE; } private static String calculateBufferKey(String fullEPR) { // cg://server1/SimpleStockQuoteService/operation1/argument1 // 5 is the length(cg://) used this way for better performance String split[] = fullEPR.substring(5).split("/"); StringBuilder buf = new StringBuilder(CGConstant.CG_TRANSPORT_PREFIX); // following is not thread safe, but there will be a thread per request // http://vanillajava.blogspot.com/2012/08/java-memes-which-refuse-to-die.html buf.append(split[0]).append("/").append(split[1]); return buf.toString(); } }