DataFlowServiceImpl.java

/*
 * *************************************************************************************
 *  Copyright (C) 2008 EsperTech, Inc. All rights reserved.                            *
 *  http://esper.codehaus.org                                                          *
 *  http://www.espertech.com                                                           *
 *  ---------------------------------------------------------------------------------- *
 *  The software in this package is published under the terms of the GPL license       *
 *  a copy of which has been included with this distribution in the license.txt file.  *
 * *************************************************************************************
 */

package com.espertech.esper.dataflow.core;

import com.espertech.esper.client.EPServiceProvider;
import com.espertech.esper.client.EPStatementState;
import com.espertech.esper.client.EventPropertyDescriptor;
import com.espertech.esper.client.EventType;
import com.espertech.esper.client.annotation.AuditEnum;
import com.espertech.esper.client.dataflow.*;
import com.espertech.esper.core.context.util.AgentInstanceContext;
import com.espertech.esper.core.service.EPRuntimeEventSender;
import com.espertech.esper.core.service.EPServicesContext;
import com.espertech.esper.core.service.StatementContext;
import com.espertech.esper.dataflow.annotations.*;
import com.espertech.esper.dataflow.interfaces.*;
import com.espertech.esper.dataflow.runnables.GraphSourceRunnable;
import com.espertech.esper.dataflow.util.*;
import com.espertech.esper.epl.annotation.AnnotationUtil;
import com.espertech.esper.epl.core.EngineImportException;
import com.espertech.esper.epl.core.EngineImportService;
import com.espertech.esper.epl.expression.ExprValidationException;
import com.espertech.esper.epl.spec.*;
import com.espertech.esper.event.EventAdapterService;
import com.espertech.esper.event.EventTypeUtility;
import com.espertech.esper.event.arr.ObjectArrayEventType;
import com.espertech.esper.util.CollectionUtil;
import com.espertech.esper.util.DependencyGraph;
import com.espertech.esper.util.JavaClassHelper;
import com.espertech.esper.util.PopulateUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.io.StringWriter;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.*;

public class DataFlowServiceImpl implements DataFlowService {
    private static final Log log = LogFactory.getLog(DataFlowServiceImpl.class);

    private static final String EVENT_WRAPPED_TYPE = "eventbean";

    private final Map<String, DataFlowServiceEntry> graphs = new HashMap<String, DataFlowServiceEntry>();
    private final Map<String, EPDataFlowInstance> instances = new HashMap<String, EPDataFlowInstance>();

    private final EPServiceProvider epService;
    private final DataFlowConfigurationStateService configurationState;

    public DataFlowServiceImpl(EPServiceProvider epService, DataFlowConfigurationStateService configurationState) {
        this.epService = epService;
        this.configurationState = configurationState;
    }

    public synchronized EPDataFlowDescriptor getDataFlow(String dataFlowName) {
        DataFlowServiceEntry entry = graphs.get(dataFlowName);
        if (entry == null) {
            return null;
        }
        return new EPDataFlowDescriptor(dataFlowName, entry.getState(), entry.getDataFlowDesc().getStatementContext().getStatementName());
    }

    public synchronized String[] getDataFlows() {
        Set<String> names = graphs.keySet();
        return names.toArray(new String[names.size()]);
    }

    public synchronized void addStartGraph(CreateDataFlowDesc desc, StatementContext statementContext, EPServicesContext servicesContext, AgentInstanceContext agentInstanceContext, boolean newStatement) throws ExprValidationException {
        compileTimeValidate(desc);

        DataFlowServiceEntry existing = graphs.get(desc.getGraphName());
        if (existing != null && (existing.getState() == EPStatementState.STARTED || newStatement)) {
            throw new ExprValidationException("Data flow by name '" + desc.getGraphName() + "' has already been declared");
        }
        if (existing != null) {
            existing.setState(EPStatementState.STARTED);
            return;
        }

        // compile annotations
        Map<GraphOperatorSpec, Annotation[]> operatorAnnotations = new HashMap<GraphOperatorSpec, Annotation[]>();
        for (GraphOperatorSpec spec : desc.getOperators()) {
            Annotation[] operatorAnnotation = AnnotationUtil.compileAnnotations(spec.getAnnotations(), servicesContext.getEngineImportService(), null);
            operatorAnnotations.put(spec, operatorAnnotation);
        }

        DataFlowStmtDesc stmtDesc = new DataFlowStmtDesc(desc, statementContext, servicesContext, agentInstanceContext, operatorAnnotations);
        graphs.put(desc.getGraphName(), new DataFlowServiceEntry(stmtDesc, EPStatementState.STARTED));
    }

    public synchronized void stopGraph(String graphName) {
        DataFlowServiceEntry existing = graphs.get(graphName);
        if (existing != null && existing.getState() == EPStatementState.STARTED) {
            existing.setState(EPStatementState.STOPPED);
        }
    }

    public synchronized void removeGraph(String graphName) {
        graphs.remove(graphName);
    }

    public EPDataFlowInstance instantiate(String dataFlowName) {
        return instantiate(dataFlowName, null);
    }

    public synchronized EPDataFlowInstance instantiate(String dataFlowName, EPDataFlowInstantiationOptions options) {
        final DataFlowServiceEntry serviceDesc = graphs.get(dataFlowName);
        if (serviceDesc == null) {
            throw new EPDataFlowInstantiationException("Data flow by name '" + dataFlowName + "' has not been defined");
        }
        if (serviceDesc.getState() != EPStatementState.STARTED) {
            throw new EPDataFlowInstantiationException("Data flow by name '" + dataFlowName + "' is currently in STOPPED statement state");
        }
        DataFlowStmtDesc stmtDesc = serviceDesc.getDataFlowDesc();
        try {
            return instantiateInternal(dataFlowName, options, stmtDesc.getGraphDesc(), stmtDesc.getStatementContext(), stmtDesc.getServicesContext(), stmtDesc.getAgentInstanceContext(), stmtDesc.getOperatorAnnotations());
        }
        catch (Exception ex) {
            String message = "Failed to instantiate data flow '" + dataFlowName + "': " + ex.getMessage();
            log.debug(message, ex);
            throw new EPDataFlowInstantiationException(message, ex);
        }
    }

    public synchronized void destroy() {
        graphs.clear();
    }

    public synchronized void saveConfiguration(String dataflowConfigName, String dataFlowName, EPDataFlowInstantiationOptions options) {
        DataFlowServiceEntry dataFlow = graphs.get(dataFlowName);
        if (dataFlow == null) {
            String message = "Failed to locate data flow '" + dataFlowName + "'";
            throw new EPDataFlowNotFoundException(message);
        }
        if (configurationState.exists(dataflowConfigName)) {
            String message = "Data flow saved configuration by name '" + dataflowConfigName + "' already exists";
            throw new EPDataFlowAlreadyExistsException(message);
        }
        configurationState.add(new EPDataFlowSavedConfiguration(dataflowConfigName, dataFlowName, options));
    }

    public synchronized String[] getSavedConfigurations() {
        return configurationState.getSavedConfigNames();
    }

    public synchronized EPDataFlowSavedConfiguration getSavedConfiguration(String configurationName) {
        return configurationState.getSavedConfig(configurationName);
    }

    public synchronized EPDataFlowInstance instantiateSavedConfiguration(String configurationName) throws EPDataFlowInstantiationException {
        EPDataFlowSavedConfiguration savedConfiguration = configurationState.getSavedConfig(configurationName);
        if (savedConfiguration == null) {
            throw new EPDataFlowInstantiationException("Dataflow saved configuration '" + configurationName + "' could not be found");
        }
        EPDataFlowInstantiationOptions options = savedConfiguration.getOptions();
        if (options == null) {
            options = new EPDataFlowInstantiationOptions();
            options.setDataFlowInstanceId(configurationName);
        }
        return instantiate(savedConfiguration.getDataflowName(), options);
    }

    public synchronized boolean removeSavedConfiguration(String configurationName) {
        return configurationState.removePrototype(configurationName) != null;
    }

    public synchronized void saveInstance(String instanceName, EPDataFlowInstance instance) throws EPDataFlowAlreadyExistsException {
        if (instances.containsKey(instanceName)) {
            throw new EPDataFlowAlreadyExistsException("Data flow instance name '" + instanceName + "' already saved");
        }
        instances.put(instanceName, instance);
    }

    public synchronized String[] getSavedInstances() {
        Set<String> instanceids = instances.keySet();
        return instanceids.toArray(new String[instanceids.size()]);
    }

    public synchronized EPDataFlowInstance getSavedInstance(String instanceName) {
        return instances.get(instanceName);
    }

    public synchronized boolean removeSavedInstance(String instanceName) {
        return instances.remove(instanceName) != null;
    }

    private EPDataFlowInstance instantiateInternal(String dataFlowName,
                                                   EPDataFlowInstantiationOptions options,
                                                   CreateDataFlowDesc desc,
                                                   StatementContext statementContext,
                                                   EPServicesContext servicesContext,
                                                   AgentInstanceContext agentInstanceContext,
                                                   Map<GraphOperatorSpec, Annotation[]> operatorAnnotations) throws ExprValidationException
    {
        if (options == null) {
            options = new EPDataFlowInstantiationOptions();
        }

        //
        // Building a model.
        //

        // resolve types
        Map<String, EventType> declaredTypes = resolveTypes(desc, statementContext, servicesContext);

        // resolve operator classes
        Map<Integer, OperatorMetadataDescriptor> operatorMetadata = resolveMetadata(desc, options, servicesContext.getEngineImportService(), operatorAnnotations);

        // build dependency graph:  operator -> [input_providing_op, input_providing_op]
        Map<Integer, OperatorDependencyEntry> operatorDependencies = analyzeDependencies(desc);

        // determine build order of operators
        Set<Integer> operatorBuildOrder = analyzeBuildOrder(operatorDependencies);

        // instantiate operators
        Map<Integer, Object> operators = instantiateOperators(operatorMetadata, desc, options, servicesContext.getEngineImportService());

        // Build graph that references port numbers (port number is simply the method offset number or to-be-generated slot in the list)
        EPRuntimeEventSender runtimeEventSender = (EPRuntimeEventSender) epService.getEPRuntime();
        List<LogicalChannel> operatorChannels = determineChannels(dataFlowName, operatorBuildOrder, operatorDependencies, operators, declaredTypes, operatorMetadata, options, servicesContext.getEventAdapterService(), servicesContext.getEngineImportService(), statementContext, servicesContext, agentInstanceContext, runtimeEventSender);
        if (log.isDebugEnabled()) {
            log.debug("For flow '" + dataFlowName + "' channels are: " + LogicalChannelUtil.printChannels(operatorChannels));
        }

        //
        // Build the realization.
        //

        // Determine binding of each channel to input methods (ports)
        List<LogicalChannelBinding> operatorChannelBindings = new ArrayList<LogicalChannelBinding>();
        for (LogicalChannel channel : operatorChannels) {
            Class targetClass = operators.get(channel.getConsumingOpNum()).getClass();
            LogicalChannelBindingMethodDesc consumingMethod = findMatchingMethod(channel.getConsumingOpPrettyPrint(), targetClass, channel, false);
            LogicalChannelBindingMethodDesc onSignalMethod = null;
            if (channel.getOutputPort().isHasPunctuation()) {
                onSignalMethod = findMatchingMethod(channel.getConsumingOpPrettyPrint(), targetClass, channel, true);
            }
            operatorChannelBindings.add(new LogicalChannelBinding(channel, consumingMethod, onSignalMethod));
        }

        // Obtain realization
        DataFlowSignalManager dataFlowSignalManager = new DataFlowSignalManager();
        DataflowStartDesc startDesc = RealizationFactoryInterface.realize(dataFlowName, operators, operatorMetadata, operatorBuildOrder, operatorChannelBindings, dataFlowSignalManager, options, servicesContext, statementContext);

        // For each GraphSource add runnable
        List<GraphSourceRunnable> sourceRunnables = new ArrayList<GraphSourceRunnable>();
        boolean audit = AuditEnum.DATAFLOW_SOURCE.getAudit(statementContext.getAnnotations()) != null;
        for (Map.Entry<Integer, Object> operatorEntry : operators.entrySet()) {
            if (!(operatorEntry.getValue() instanceof DataFlowSourceOperator)) {
                continue;
            }
            OperatorMetadataDescriptor meta = operatorMetadata.get(operatorEntry.getKey());

            DataFlowSourceOperator graphSource = (DataFlowSourceOperator) operatorEntry.getValue();
            GraphSourceRunnable runnable = new GraphSourceRunnable(statementContext.getEngineURI(), statementContext.getStatementName(), graphSource, dataFlowName, meta.getOperatorName(), operatorEntry.getKey(), meta.getOperatorPrettyPrint(), options.getExceptionHandler(), audit);
            sourceRunnables.add(runnable);
            
            dataFlowSignalManager.addSignalListener(operatorEntry.getKey(), runnable);
        }

        boolean auditStates = AuditEnum.DATAFLOW_TRANSITION.getAudit(statementContext.getAnnotations()) != null;
        return new EPDataFlowInstanceImpl(servicesContext.getEngineURI(), statementContext.getStatementName(), auditStates, dataFlowName, options.getDataFlowInstanceUserObject(), options.getDataFlowInstanceId(), EPDataFlowState.INSTANTIATED, sourceRunnables, operators, operatorBuildOrder, startDesc.getStatisticsProvider());
    }

    private Map<String, EventType> resolveTypes(CreateDataFlowDesc desc, StatementContext statementContext, EPServicesContext servicesContext)
            throws ExprValidationException
    {
        Map<String, EventType> types = new HashMap<String, EventType>();
        for (CreateSchemaDesc spec : desc.getSchemas()) {
            EventType eventType = EventTypeUtility.createNonVariantType(true, spec, statementContext.getAnnotations(), statementContext.getConfigSnapshot(),
                                statementContext.getEventAdapterService(), servicesContext.getEngineImportService());
            types.put(spec.getSchemaName(), eventType);
        }
        return types;
    }

    private Map<Integer, Object> instantiateOperators(Map<Integer, OperatorMetadataDescriptor> operatorClasses, CreateDataFlowDesc desc, EPDataFlowInstantiationOptions options, EngineImportService engineImportService)
        throws ExprValidationException
    {
        Map<Integer, Object> operators = new HashMap<Integer, Object>();

        for (Map.Entry<Integer, OperatorMetadataDescriptor> operatorEntry : operatorClasses.entrySet()) {

            Object operator = instantiateOperator(desc.getGraphName(), operatorEntry.getKey(), operatorEntry.getValue(), desc.getOperators().get(operatorEntry.getKey()), options, engineImportService);
            operators.put(operatorEntry.getKey(), operator);
        }

        return operators;
    }

    private Object instantiateOperator(String dataFlowName, int operatorNum, OperatorMetadataDescriptor desc, GraphOperatorSpec graphOperator, EPDataFlowInstantiationOptions options, EngineImportService engineImportService)
        throws ExprValidationException {

        Object operatorObject = desc.getOptionalOperatorObject();

        if (operatorObject == null) {
            Class clazz = desc.getOperatorFactoryClass() != null ? desc.getOperatorFactoryClass() : desc.getOperatorClass();

            // use non-factory class if provided
            try {
                operatorObject = clazz.newInstance();
            }
            catch (Exception e) {
                throw new ExprValidationException("Failed to instantiate: " + e.getMessage());
            }
        }

        // inject properties
        Map<String, Object> configs = graphOperator.getDetail() == null ? Collections.<String, Object>emptyMap() : graphOperator.getDetail().getConfigs();
        injectObjectProperties(dataFlowName, graphOperator.getOperatorName(), operatorNum, configs, operatorObject, options.getParameterProvider(), options.getParametersURIs(), engineImportService);

        if (operatorObject instanceof DataFlowOperatorFactory) {
            try {
                operatorObject = ((DataFlowOperatorFactory) operatorObject).create();
            }
            catch (RuntimeException ex) {
                throw new ExprValidationException("Failed to obtain operator '" + desc.getOperatorName() + "', encountered an exception raised by factory class " + operatorObject.getClass().getSimpleName() + ": " + ex.getMessage(), ex);
            }
        }

        return operatorObject;
    }

    private void injectObjectProperties(String dataFlowName, String operatorName, int operatorNum, Map<String, Object> configs, Object instance, EPDataFlowOperatorParameterProvider optionalParameterProvider, Map<String, Object> optionalParameterURIs, EngineImportService engineImportService)
        throws ExprValidationException {

        // determine if there is a property holder which holds all properties
        Set<Field> propertyHolderFields = JavaClassHelper.findAnnotatedFields(instance.getClass(), DataFlowOpPropertyHolder.class);
        if (propertyHolderFields.size() > 1) {
            throw new IllegalArgumentException("May apply " + DataFlowOpPropertyHolder.class.getSimpleName() + " annotation only to a single field");
        }

        // determine which class to write properties to
        Object propertyInstance;
        if (propertyHolderFields.isEmpty()) {
            propertyInstance = instance;
        }
        else {
            Class propertyHolderClass = propertyHolderFields.iterator().next().getType();
            try {
                propertyInstance = propertyHolderClass.newInstance();
            }
            catch (Exception e) {
                throw new ExprValidationException("Failed to instantiate '" + propertyHolderClass + "': " + e.getMessage(), e);
            }
        }

        // populate either the instance itself or the property-holder
        PopulateUtil.populateObject(operatorName, operatorNum, dataFlowName, configs, propertyInstance, engineImportService, optionalParameterProvider, optionalParameterURIs);

        // set holder
        if (!propertyHolderFields.isEmpty()) {
            Field field = propertyHolderFields.iterator().next();
            try {
                field.setAccessible(true);
                field.set(instance, propertyInstance);
            }
            catch (Exception e) {
                throw new ExprValidationException("Failed to set field '" + field.getName() + "': " + e.getMessage(), e);
            }
        }
    }

    private List<LogicalChannel> determineChannels(String dataflowName,
                                                       Set<Integer> operatorBuildOrder,
                                                       Map<Integer, OperatorDependencyEntry> operatorDependencies,
                                                       Map<Integer, Object> operators,
                                                       Map<String, EventType> types,
                                                       Map<Integer, OperatorMetadataDescriptor> operatorMetadata,
                                                       EPDataFlowInstantiationOptions options,
                                                       EventAdapterService eventAdapterService,
                                                       EngineImportService engineImportService,
                                                       StatementContext statementContext,
                                                       EPServicesContext servicesContext,
                                                       AgentInstanceContext agentInstanceContext,
                                                       EPRuntimeEventSender runtimeEventSender)
        throws ExprValidationException
    {
        // This is a multi-step process.
        //
        // Step 1: find all the operators that have explicit output ports and determine the type of such
        Map<Integer, List<LogicalChannelProducingPortDeclared>> declaredOutputPorts = new HashMap<Integer, List<LogicalChannelProducingPortDeclared>>();
        for (int operatorNum : operatorBuildOrder) {

            OperatorMetadataDescriptor metadata = operatorMetadata.get(operatorNum);
            Object operator = operators.get(operatorNum);

            List<LogicalChannelProducingPortDeclared> annotationPorts = determineAnnotatedOutputPorts(operatorNum, operator, metadata, engineImportService, eventAdapterService);
            List<LogicalChannelProducingPortDeclared> graphDeclaredPorts = determineGraphDeclaredOutputPorts(operator, operatorNum, metadata, types, servicesContext);

            List<LogicalChannelProducingPortDeclared> allDeclaredPorts = new ArrayList<LogicalChannelProducingPortDeclared>();
            allDeclaredPorts.addAll(annotationPorts);
            allDeclaredPorts.addAll(graphDeclaredPorts);

            declaredOutputPorts.put(operatorNum, allDeclaredPorts);
        }

        // Step 2: determine for each operator the output ports: some are determined via "prepare" and some can be implicit
        // since they may not be declared or can be punctuation.
        // Therefore we need to meet ends: on one end the declared types, on the other the implied and dynamically-determined types based on input.
        // We do this in operator build order.
        Map<Integer, List<LogicalChannelProducingPortCompiled>> compiledOutputPorts = new HashMap<Integer, List<LogicalChannelProducingPortCompiled>>();
        for (int myOpNum : operatorBuildOrder) {

            GraphOperatorSpec operatorSpec = operatorMetadata.get(myOpNum).getOperatorSpec();
            Object operator = operators.get(myOpNum);
            OperatorMetadataDescriptor metadata = operatorMetadata.get(myOpNum);

            // Handle incoming first: if the operator has incoming ports, each of such should already have type information
            // Compile type information, call method, obtain output types.
            Set<Integer> incomingDependentOpNums = operatorDependencies.get(myOpNum).getIncoming();
            GraphTypeDesc[] typesPerOutput = determineOutputForInput(dataflowName, myOpNum, operator, metadata, operatorSpec, declaredOutputPorts, compiledOutputPorts, types, incomingDependentOpNums, options, statementContext, servicesContext, agentInstanceContext, runtimeEventSender);

            // Handle outgoing second:
            //   If there is outgoing declared, use that.
            //   If output types have been determined based on input, use that.
            //   else error
            List<LogicalChannelProducingPortCompiled> outgoingPorts = determineOutgoingPorts(myOpNum, operator, operatorSpec, metadata, compiledOutputPorts, declaredOutputPorts, typesPerOutput, incomingDependentOpNums);
            compiledOutputPorts.put(myOpNum, outgoingPorts);
        }

        // Step 3: normalization and connecting input ports with output ports (logically, no methods yet)
        List<LogicalChannel> channels = new ArrayList<LogicalChannel>();
        int channelId = 0;
        for (Integer myOpNum : operatorBuildOrder) {

            OperatorDependencyEntry dependencies = operatorDependencies.get(myOpNum);
            List<GraphOperatorInputNamesAlias> inputNames = operatorMetadata.get(myOpNum).getOperatorSpec().getInput().getStreamNamesAndAliases();
            OperatorMetadataDescriptor descriptor = operatorMetadata.get(myOpNum);

            // handle each (a,b,c AS d)
            int streamNum = -1;
            for (GraphOperatorInputNamesAlias inputName : inputNames) {
                streamNum++;

                // get producers
                List<LogicalChannelProducingPortCompiled> producingPorts = LogicalChannelUtil.getOutputPortByStreamName(dependencies.getIncoming(), inputName.getInputStreamNames(), compiledOutputPorts);
                if (producingPorts.size() < inputName.getInputStreamNames().length) {
                    throw new IllegalStateException("Failed to find producing ports");
                }

                // determine type compatibility
                if (producingPorts.size() > 1) {
                    LogicalChannelProducingPortCompiled first = producingPorts.get(0);
                    for (int i = 1; i < producingPorts.size(); i++) {
                        LogicalChannelProducingPortCompiled other = producingPorts.get(i);
                        compareTypeInfo(descriptor.getOperatorName(), first.getStreamName(), first.getGraphTypeDesc(), other.getStreamName(), other.getGraphTypeDesc());
                    }
                }

                String optionalAlias = inputName.getOptionalAsName();

                // handle each stream name
                for (String streamName : inputName.getInputStreamNames()) {

                    for (LogicalChannelProducingPortCompiled port : producingPorts) {
                        if (port.getStreamName().equals(streamName)) {
                            LogicalChannel channel = new LogicalChannel(channelId++, descriptor.getOperatorName(), myOpNum, streamNum, streamName, optionalAlias, descriptor.getOperatorPrettyPrint(), port);
                            channels.add(channel);
                        }
                    }
                }
            }
        }

        return channels;
    }

    private void compareTypeInfo(String operatorName, String firstName, GraphTypeDesc firstType, String otherName, GraphTypeDesc otherType)
        throws ExprValidationException
    {
        if (firstType.getEventType() != null && otherType.getEventType() != null && !firstType.getEventType().equals(otherType.getEventType())) {
            throw new ExprValidationException("For operator '" + operatorName + "' stream '" + firstName + "'" +
                    " typed '" + firstType.getEventType().getName() + "'" +
                    " is not the same type as stream '" + otherName + "'" +
                    " typed '" + otherType.getEventType().getName() + "'");
        }
        if (firstType.isWildcard() != otherType.isWildcard()) {
            throw new ExprValidationException("For operator '" + operatorName + "' streams '" + firstName + "'" +
                    " and '" + otherName + "' have differing wildcard type information");
        }
        if (firstType.isUnderlying() != otherType.isUnderlying()) {
            throw new ExprValidationException("For operator '" + operatorName + "' streams '" + firstName + "'" +
                    " and '" + otherName + "' have differing underlying information");
        }
    }

    private List<LogicalChannelProducingPortCompiled> determineOutgoingPorts(int myOpNum,
                                                                             Object operator,
                                                                             GraphOperatorSpec operatorSpec,
                                                                             OperatorMetadataDescriptor metadata,
                                                                             Map<Integer, List<LogicalChannelProducingPortCompiled>> compiledOutputPorts,
                                                                             Map<Integer, List<LogicalChannelProducingPortDeclared>> declaredOutputPorts,
                                                                             GraphTypeDesc[] typesPerOutput,
                                                                             Set<Integer> incomingDependentOpNums)
        throws ExprValidationException
    {
        // Either
        //  (A) the port is explicitly declared via @OutputTypes
        //  (B) the port is declared via "=> ABC<type>"
        //  (C) the port is implicit since there is only one input port and the operator is a functor

        int numPorts = operatorSpec.getOutput().getItems().size();
        List<LogicalChannelProducingPortCompiled> result = new ArrayList<LogicalChannelProducingPortCompiled>();

        // we go port-by-port: what was declared, what types were determined
        Map<String, GraphTypeDesc> types = new HashMap<String, GraphTypeDesc>();
        for (int port = 0; port < numPorts; port++ ) {
            String portStreamName = operatorSpec.getOutput().getItems().get(port).getStreamName();

            // find declaration, if any
            LogicalChannelProducingPortDeclared foundDeclared = null;
            List<LogicalChannelProducingPortDeclared> declaredList = declaredOutputPorts.get(myOpNum);
            for (LogicalChannelProducingPortDeclared declared : declaredList) {
                if (declared.getStreamNumber() == port) {
                    if (foundDeclared != null) {
                        throw new ExprValidationException("Found a declaration twice for port " + port);
                    }
                    foundDeclared = declared;
                }
            }

            if (foundDeclared == null && (typesPerOutput == null || typesPerOutput.length <= port || typesPerOutput[port] == null)) {
                throw new ExprValidationException("Operator neither declares an output type nor provided by the operator itself in a 'prepare' method");
            }
            if (foundDeclared != null && typesPerOutput != null && typesPerOutput.length > port && typesPerOutput[port] != null) {
                throw new ExprValidationException("Operator both declares an output type and provided a type in the 'prepare' method");
            }

            // punctuation determined by input
            boolean hasPunctuationSignal = (foundDeclared != null ? foundDeclared.isHasPunctuation() : false) || determineReceivesPunctuation(incomingDependentOpNums, operatorSpec.getInput(), compiledOutputPorts);

            GraphTypeDesc compiledType;
            if (foundDeclared != null) {
                compiledType = foundDeclared.getTypeDesc();
            }
            else {
                compiledType = typesPerOutput[port];
            }

            LogicalChannelProducingPortCompiled compiled = new LogicalChannelProducingPortCompiled(myOpNum, metadata.getOperatorPrettyPrint(), portStreamName, port, compiledType, hasPunctuationSignal);
            result.add(compiled);

            // check type compatibility
            GraphTypeDesc existingType = types.get(portStreamName);
            types.put(portStreamName, compiledType);
            if (existingType != null) {
                compareTypeInfo(operatorSpec.getOperatorName(), portStreamName, existingType, portStreamName, compiledType);
            }
        }

        return result;
    }

    private boolean determineReceivesPunctuation(Set<Integer> incomingDependentOpNums, GraphOperatorInput input, Map<Integer, List<LogicalChannelProducingPortCompiled>> compiledOutputPorts) {
        for (GraphOperatorInputNamesAlias inputItem : input.getStreamNamesAndAliases()) {
            List<LogicalChannelProducingPortCompiled> list = LogicalChannelUtil.getOutputPortByStreamName(incomingDependentOpNums, inputItem.getInputStreamNames(), compiledOutputPorts);
            for (LogicalChannelProducingPortCompiled port : list) {
                if (port.isHasPunctuation()) {
                    return true;
                }
            }
        }
        return false;
    }

    private GraphTypeDesc[] determineOutputForInput(String dataFlowName,
                                                    int myOpNum,
                                                Object operator,
                                                OperatorMetadataDescriptor meta,
                                                GraphOperatorSpec operatorSpec,
                                                Map<Integer, List<LogicalChannelProducingPortDeclared>> declaredOutputPorts,
                                                Map<Integer, List<LogicalChannelProducingPortCompiled>> compiledOutputPorts,
                                                Map<String, EventType> types,
                                                Set<Integer> incomingDependentOpNums,
                                                EPDataFlowInstantiationOptions options,
                                                StatementContext statementContext,
                                                EPServicesContext servicesContext,
                                                AgentInstanceContext agentInstanceContext,
                                                EPRuntimeEventSender runtimeEventSender)
        throws ExprValidationException
    {
        if (!(operator instanceof DataFlowOpLifecycle)) {
            return null;
        }

        // determine input ports to build up the input port metadata
        int numDeclared = operatorSpec.getInput().getStreamNamesAndAliases().size();
        Map<Integer, DataFlowOpInputPort> inputPorts = new LinkedHashMap<Integer, DataFlowOpInputPort>();
        for (int inputPortNum = 0; inputPortNum < numDeclared; inputPortNum++) {
            GraphOperatorInputNamesAlias inputItem = operatorSpec.getInput().getStreamNamesAndAliases().get(inputPortNum);
            List<LogicalChannelProducingPortCompiled> producingPorts = LogicalChannelUtil.getOutputPortByStreamName(incomingDependentOpNums, inputItem.getInputStreamNames(), compiledOutputPorts);

            DataFlowOpInputPort port;
            if (producingPorts.isEmpty()) { // this can be when the operator itself is the incoming port, i.e. feedback loop
                List<LogicalChannelProducingPortDeclared> declareds = declaredOutputPorts.get(myOpNum);
                if (declareds == null || declareds.isEmpty()) {
                    throw new ExprValidationException("Failed validation for operator '" + operatorSpec.getOperatorName() + "': No output ports declared");
                }
                LogicalChannelProducingPortDeclared foundDeclared = null;
                for (LogicalChannelProducingPortDeclared declared : declareds) {
                    if (Arrays.asList(inputItem.getInputStreamNames()).contains(declared.getStreamName())) {
                        foundDeclared = declared;
                        break;
                    }
                }
                if (foundDeclared == null) {
                    throw new ExprValidationException("Failed validation for operator '" + operatorSpec.getOperatorName() + "': Failed to find output port declared");
                }
                port = new DataFlowOpInputPort(foundDeclared.getTypeDesc(), new HashSet<String>(Arrays.asList(inputItem.getInputStreamNames())), inputItem.getOptionalAsName(), false);
            }
            else {
                port = new DataFlowOpInputPort(new GraphTypeDesc(false, false, producingPorts.get(0).getGraphTypeDesc().getEventType()), new HashSet<String>(Arrays.asList(inputItem.getInputStreamNames())), inputItem.getOptionalAsName(), producingPorts.get(0).isHasPunctuation());
            }
            inputPorts.put(inputPortNum, port);
        }

        // determine output ports to build up the output port metadata
        Map<Integer, DataFlowOpOutputPort> outputPorts = getDeclaredOutputPorts(operatorSpec, types, servicesContext);

        // determine event sender
        EPRuntimeEventSender dfRuntimeEventSender = runtimeEventSender;
        if (options.getSurrogateEventSender() != null) {
            dfRuntimeEventSender = options.getSurrogateEventSender();
        }

        DataFlowOpLifecycle preparable = (DataFlowOpLifecycle) operator;
        DataFlowOpInitializateContext context = new DataFlowOpInitializateContext(dataFlowName, options.getDataFlowInstanceId(), options.getDataFlowInstanceUserObject(), inputPorts, outputPorts, statementContext, servicesContext, agentInstanceContext, dfRuntimeEventSender, epService, meta.getOperatorAnnotations());

        DataFlowOpInitializeResult prepareResult;
        try {
            prepareResult = preparable.initialize(context);
        }
        catch (ExprValidationException e) {
            throw new ExprValidationException("Failed validation for operator '" + operatorSpec.getOperatorName() + "': " + e.getMessage(), e);
        }
        catch (Exception e) {
            throw new ExprValidationException("Failed initialization for operator '" + operatorSpec.getOperatorName() + "': " + e.getMessage(), e);
        }

        if (prepareResult == null) {
            return null;
        }
        return prepareResult.getTypeDescriptors();
    }

    private List<LogicalChannelProducingPortDeclared> determineAnnotatedOutputPorts(int producingOpNum, Object operator, OperatorMetadataDescriptor descriptor, EngineImportService engineImportService, EventAdapterService eventAdapterService)
        throws ExprValidationException {

        List<LogicalChannelProducingPortDeclared> ports = new ArrayList<LogicalChannelProducingPortDeclared>();

        // See if any @OutputTypes annotations exists
        List<Annotation> annotations = JavaClassHelper.getAnnotations(OutputTypes.class, operator.getClass().getDeclaredAnnotations());

        for (Annotation annotation : annotations) {

            OutputTypes outputTypes = (OutputTypes) annotation;

            // create local event type for the declared type
            Map<String, Object> propertiesRaw = new LinkedHashMap<String, Object>();
            OutputType[] outputTypeArr = outputTypes.value();
            for (OutputType outputType : outputTypeArr) {
                Class clazz;
                if (outputType.type() != null && outputType.type() != OutputType.class) {
                    clazz = outputType.type();
                }
                else {
                    String typeName = outputType.typeName();
                    clazz = JavaClassHelper.getClassForSimpleName(typeName);
                    if (clazz == null) {
                        try {
                            clazz = engineImportService.resolveClass(typeName);
                        }
                        catch (EngineImportException e) {
                            throw new RuntimeException("Failed to resolve type '" + typeName + "'");
                        }
                    }
                }
                propertiesRaw.put(outputType.name(), clazz);
            }
            Map<String, Object> propertiesCompiled = EventTypeUtility.compileMapTypeProperties(propertiesRaw, eventAdapterService);
            EventType eventType = eventAdapterService.createAnonymousObjectArrayType("TYPE_" + operator.getClass(), propertiesCompiled);

            // determine output stream name, which must be provided
            List<GraphOperatorOutputItem> declaredOutput = descriptor.getOperatorSpec().getOutput().getItems();
            if (declaredOutput.isEmpty()) {
                throw new ExprValidationException("No output stream declared");
            }
            if (declaredOutput.size() < outputTypes.portNumber()) {
                throw new ExprValidationException("No output stream declared for this port");
            }
            String streamName = declaredOutput.get(outputTypes.portNumber()).getStreamName();

            boolean isDeclaredPunctuated = JavaClassHelper.isAnnotationListed(DataFlowOpProvideSignal.class, operator.getClass().getAnnotations());
            LogicalChannelProducingPortDeclared port = new LogicalChannelProducingPortDeclared(producingOpNum, descriptor.getOperatorPrettyPrint(), streamName, outputTypes.portNumber(), new GraphTypeDesc(false, false, eventType), isDeclaredPunctuated);
            ports.add(port);
        }

        return ports;
    }

    private List<LogicalChannelProducingPortDeclared> determineGraphDeclaredOutputPorts(Object operator, int producingOpNum, OperatorMetadataDescriptor metadata, Map<String, EventType> types, EPServicesContext servicesContext)
        throws ExprValidationException {

        List<LogicalChannelProducingPortDeclared> ports = new ArrayList<LogicalChannelProducingPortDeclared>();

        int portNumber = 0;
        for (GraphOperatorOutputItem outputItem : metadata.getOperatorSpec().getOutput().getItems()) {
            if (outputItem.getTypeInfo().size() > 1) {
                throw new ExprValidationException("Multiple parameter types are not supported");
            }

            if (!outputItem.getTypeInfo().isEmpty()) {
                GraphTypeDesc typeDesc = determineTypeOutputPort(outputItem.getTypeInfo().get(0), types, servicesContext);
                boolean isDeclaredPunctuated = JavaClassHelper.isAnnotationListed(DataFlowOpProvideSignal.class, operator.getClass().getAnnotations());
                ports.add(new LogicalChannelProducingPortDeclared(producingOpNum, metadata.getOperatorPrettyPrint(), outputItem.getStreamName(), portNumber, typeDesc, isDeclaredPunctuated));
            }
            portNumber++;
        }

        return ports;
    }

    private Map<Integer, OperatorDependencyEntry> analyzeDependencies(CreateDataFlowDesc graphDesc)
            throws ExprValidationException
    {
        Map<Integer, OperatorDependencyEntry> logicalOpDependencies = new HashMap<Integer, OperatorDependencyEntry>();
        for (int i = 0; i < graphDesc.getOperators().size(); i++) {
            OperatorDependencyEntry entry = new OperatorDependencyEntry();
            logicalOpDependencies.put(i, entry);
        }
        for (int consumingOpNum = 0; consumingOpNum < graphDesc.getOperators().size(); consumingOpNum++) {
            OperatorDependencyEntry entry = logicalOpDependencies.get(consumingOpNum);
            GraphOperatorSpec op = graphDesc.getOperators().get(consumingOpNum);

            // for each input item
            for (GraphOperatorInputNamesAlias input : op.getInput().getStreamNamesAndAliases()) {

                // for each stream name listed
                for (String inputStreamName : input.getInputStreamNames()) {

                    // find all operators providing such input stream
                    boolean found = false;

                    // for each operator
                    for (int providerOpNum = 0; providerOpNum < graphDesc.getOperators().size(); providerOpNum++) {
                        GraphOperatorSpec from = graphDesc.getOperators().get(providerOpNum);

                        for (GraphOperatorOutputItem outputItem : from.getOutput().getItems()) {
                            if (outputItem.getStreamName().equals(inputStreamName)) {
                                found = true;
                                entry.addIncoming(providerOpNum);
                                logicalOpDependencies.get(providerOpNum).addOutgoing(consumingOpNum);
                            }
                        }
                    }

                    if (!found) {
                        throw new ExprValidationException("Input stream '" + inputStreamName + "' consumed by operator '" + op.getOperatorName() + "' could not be found");
                    }
                }
            }
        }
        return logicalOpDependencies;
    }

    private Map<Integer, OperatorMetadataDescriptor> resolveMetadata(CreateDataFlowDesc graphDesc, EPDataFlowInstantiationOptions options, EngineImportService engineImportService, Map<GraphOperatorSpec, Annotation[]> operatorAnnotations)
        throws ExprValidationException
    {
        Map<Integer, OperatorMetadataDescriptor> operatorClasses = new HashMap<Integer, OperatorMetadataDescriptor>();
        for (int i = 0; i < graphDesc.getOperators().size(); i++) {
            GraphOperatorSpec operatorSpec = graphDesc.getOperators().get(i);
            String operatorPrettyPrint = toPrettyPrint(i, operatorSpec);
            Annotation[] operatorAnnotation = operatorAnnotations.get(operatorSpec);

            // see if the operator is already provided by options
            if (options.getOperatorProvider() != null) {
                Object operator = options.getOperatorProvider().provide(new EPDataFlowOperatorProviderContext(graphDesc.getGraphName(), operatorSpec.getOperatorName(), operatorSpec));
                if (operator != null) {
                    OperatorMetadataDescriptor descriptor = new OperatorMetadataDescriptor(operatorSpec, i, operator.getClass(), null, operator, operatorPrettyPrint, operatorAnnotation);
                    operatorClasses.put(i, descriptor);
                    continue;
                }
            }

            // try to find factory class with factory annotation
            Class factoryClass = null;
            try {
                factoryClass = engineImportService.resolveClass(operatorSpec.getOperatorName() + "Factory");
            }
            catch (EngineImportException e) {
            }

            // if the factory implements the interface use that
            if (factoryClass != null && JavaClassHelper.isImplementsInterface(factoryClass, DataFlowOperatorFactory.class)) {
                OperatorMetadataDescriptor descriptor = new OperatorMetadataDescriptor(operatorSpec, i, null, factoryClass, null, operatorPrettyPrint, operatorAnnotation);
                operatorClasses.put(i, descriptor);
                continue;
            }

            // resolve by class name
            Class clazz;
            try {
                clazz = engineImportService.resolveClass(operatorSpec.getOperatorName());
            }
            catch (EngineImportException e) {
                throw new ExprValidationException("Failed to resolve operator '" + operatorSpec.getOperatorName() + "': " + e.getMessage(), e);
            }

            if (!JavaClassHelper.isImplementsInterface(clazz, DataFlowSourceOperator.class) &&
                !JavaClassHelper.isAnnotationListed(DataFlowOperator.class, clazz.getDeclaredAnnotations())) {
                throw new ExprValidationException("Failed to resolve operator '" + operatorSpec.getOperatorName() + "', operator class " + clazz.getName() + " does not declare the " + DataFlowOperator.class.getSimpleName() + " annotation or implement the " + DataFlowSourceOperator.class.getSimpleName() + " interface");
            }

            OperatorMetadataDescriptor descriptor = new OperatorMetadataDescriptor(operatorSpec, i, clazz, null, null, operatorPrettyPrint, operatorAnnotation);
            operatorClasses.put(i, descriptor);
        }
        return operatorClasses;
    }

    private String toPrettyPrint(int operatorNum, GraphOperatorSpec spec) {
        StringWriter writer = new StringWriter();
        writer.write(spec.getOperatorName());
        writer.write("#");
        writer.write(Integer.toString(operatorNum));

        writer.write("(");
        String delimiter = "";
        for (GraphOperatorInputNamesAlias inputItem : spec.getInput().getStreamNamesAndAliases()) {
            writer.write(delimiter);
            toPrettyPrintInput(inputItem, writer);
            if (inputItem.getOptionalAsName() != null) {
                writer.write(" as ");
                writer.write(inputItem.getOptionalAsName());
            }
            delimiter = ", ";
        }
        writer.write(")");

        if (spec.getOutput().getItems().isEmpty()) {
            return writer.toString();
        }
        writer.write(" -> ");

        delimiter = "";
        for (GraphOperatorOutputItem outputItem : spec.getOutput().getItems()) {
            writer.write(delimiter);
            writer.write(outputItem.getStreamName());
            writeTypes(outputItem.getTypeInfo(), writer);
            delimiter = ",";
        }

        return writer.toString();
    }

    private void toPrettyPrintInput(GraphOperatorInputNamesAlias inputItem, StringWriter writer) {
        if (inputItem.getInputStreamNames().length == 1) {
            writer.write(inputItem.getInputStreamNames()[0]);
        }
        else {
            writer.write("(");
            String delimiterNames = "";
            for (String name : inputItem.getInputStreamNames()) {
                writer.write(delimiterNames);
                writer.write(name);
                delimiterNames = ",";
            }
            writer.write(")");
        }
}

    private void writeTypes(List<GraphOperatorOutputItemType> types, StringWriter writer) {
        if (types.isEmpty()) {
            return;
        }

        writer.write("<");
        String typeDelimiter = "";
        for (GraphOperatorOutputItemType type : types) {
            writer.write(typeDelimiter);
            writeType(type, writer);
            typeDelimiter = ",";
        }
        writer.write(">");
    }

    private void writeType(GraphOperatorOutputItemType type, StringWriter writer) {
        if (type.isWildcard()) {
            writer.append('?');
            return;
        }
        writer.append(type.getTypeOrClassname());
        writeTypes(type.getTypeParameters(), writer);
    }

    private Set<Integer> analyzeBuildOrder(Map<Integer, OperatorDependencyEntry> operators) throws ExprValidationException {

        DependencyGraph graph = new DependencyGraph(operators.size(), true);
        for (Map.Entry<Integer, OperatorDependencyEntry> entry : operators.entrySet()) {
            int myOpNum = entry.getKey();
            Set<Integer> incomings = entry.getValue().getIncoming();
            for (int incoming : incomings) {
                graph.addDependency(myOpNum, incoming);
            }
        }

        Set<Integer> topDownSet = new HashSet<Integer>();
        while(topDownSet.size() < operators.size()) {

            // seconardy sort according to the order of listing
            Set<Integer> rootNodes = new TreeSet<Integer>(new Comparator<Integer>() {
                public int compare(Integer o1, Integer o2)
                {
                    return -1 * o1.compareTo(o2);
                }
            });
            rootNodes.addAll(graph.getRootNodes(topDownSet));

            if (rootNodes.isEmpty()) {   // circular dependency could cause this
                for (int i = 0; i < operators.size(); i++) {
                    if (!topDownSet.contains(i)) {
                        rootNodes.add(i);
                        break;
                    }
                }
            }

            topDownSet.addAll(rootNodes);
        }

        return topDownSet;
    }

    private LogicalChannelBindingMethodDesc findMatchingMethod(String operatorName, Class target, LogicalChannel channelDesc, boolean isPunctuation)
        throws ExprValidationException
    {
        if (isPunctuation) {
            for (Method method : target.getMethods()) {
                if (method.getName().equals("onSignal")) {
                    return new LogicalChannelBindingMethodDesc(method, LogicalChannelBindingTypePassAlong.INSTANCE);
                }
            }
            return null;
        }

        LogicalChannelProducingPortCompiled outputPort = channelDesc.getOutputPort();

        Class[] expectedIndividual;
        Class expectedUnderlying;
        EventType expectedUnderlyingType;
        GraphTypeDesc typeDesc = outputPort.getGraphTypeDesc();

        if (typeDesc.isWildcard()) {
            expectedIndividual = new Class[0];
            expectedUnderlying = null;
            expectedUnderlyingType = null;
        }
        else {
            expectedIndividual = new Class[typeDesc.getEventType().getPropertyNames().length];
            int i = 0;
            for (EventPropertyDescriptor descriptor : typeDesc.getEventType().getPropertyDescriptors()) {
                expectedIndividual[i] = descriptor.getPropertyType();
                i++;
            }
            expectedUnderlying = typeDesc.getEventType().getUnderlyingType();
            expectedUnderlyingType = typeDesc.getEventType();
        }

        String channelSpecificMethodName = null;
        if (channelDesc.getConsumingOptStreamAliasName() != null) {
            channelSpecificMethodName = "on" + channelDesc.getConsumingOptStreamAliasName();
        }

        for (Method method : target.getMethods()) {

            boolean eligible = method.getName().equals("onInput");
            if (!eligible && method.getName().equals(channelSpecificMethodName)) {
                eligible = true;
            }

            if (!eligible ) {
                continue;
            }

            // handle Object[]
            int numParams = method.getParameterTypes().length;
            Class[] paramTypes = method.getParameterTypes();

            if (expectedUnderlying != null) {
                if (numParams == 1 && JavaClassHelper.isSubclassOrImplementsInterface(paramTypes[0], expectedUnderlying)) {
                    return new LogicalChannelBindingMethodDesc(method, LogicalChannelBindingTypePassAlong.INSTANCE);
                }
                if (numParams == 2 && JavaClassHelper.getBoxedType(paramTypes[0]) == Integer.class && JavaClassHelper.isSubclassOrImplementsInterface(paramTypes[1], expectedUnderlying)) {
                    return new LogicalChannelBindingMethodDesc(method, new LogicalChannelBindingTypePassAlongWStream(channelDesc.getConsumingOpStreamNum()));
                }
            }

            if (numParams == 1 && (paramTypes[0] == Object.class || (paramTypes[0] == Object[].class && method.isVarArgs()))) {
                return new LogicalChannelBindingMethodDesc(method, LogicalChannelBindingTypePassAlong.INSTANCE);
            }
            if (numParams == 2 && paramTypes[0] == int.class && (paramTypes[1] == Object.class || (paramTypes[1] == Object[].class && method.isVarArgs()))) {
                return new LogicalChannelBindingMethodDesc(method, new LogicalChannelBindingTypePassAlongWStream(channelDesc.getConsumingOpStreamNum()));
            }

            // if exposing a method that exactly matches each property type in order, use that, i.e. "onInut(String p0, int p1)"
            if (expectedUnderlyingType instanceof ObjectArrayEventType && JavaClassHelper.isSignatureCompatible(expectedIndividual, method.getParameterTypes())) {
                return new LogicalChannelBindingMethodDesc(method, LogicalChannelBindingTypeUnwind.INSTANCE);
            }
        }

        Set<String> choices = new LinkedHashSet<String>();
        choices.add(Object.class.getSimpleName());
        choices.add("Object[]");
        if (expectedUnderlying != null) {
            choices.add(expectedUnderlying.getSimpleName());
        }
        throw new ExprValidationException("Failed to find onInput method on for operator '" + operatorName + "' class " +
                target.getName() + ", expected an onInput method that takes any of {" + CollectionUtil.toString(choices) + "}");
    }

    private static Map<Integer,DataFlowOpOutputPort> getDeclaredOutputPorts(GraphOperatorSpec operatorSpec, Map<String, EventType> types, EPServicesContext servicesContext)
        throws ExprValidationException
    {
        Map<Integer,DataFlowOpOutputPort> outputPorts = new LinkedHashMap<Integer, DataFlowOpOutputPort>();
        for (int outputPortNum = 0; outputPortNum < operatorSpec.getOutput().getItems().size(); outputPortNum++) {
            GraphOperatorOutputItem outputItem = operatorSpec.getOutput().getItems().get(outputPortNum);

            GraphTypeDesc typeDesc = null;
            if (!outputItem.getTypeInfo().isEmpty()) {
                typeDesc = determineTypeOutputPort(outputItem.getTypeInfo().get(0), types, servicesContext);
            }
            outputPorts.put(outputPortNum, new DataFlowOpOutputPort(outputItem.getStreamName(), typeDesc));
        }

        return outputPorts;
    }

    private static GraphTypeDesc determineTypeOutputPort(GraphOperatorOutputItemType outType, Map<String, EventType> types, EPServicesContext servicesContext)
        throws ExprValidationException {
        
        EventType eventType = null;
        boolean isWildcard = false;
        boolean isUnderlying = true;

        String typeOrClassname = outType.getTypeOrClassname();
        if (typeOrClassname != null && typeOrClassname.toLowerCase().equals(EVENT_WRAPPED_TYPE)) {
            isUnderlying = false;
            if (!outType.getTypeParameters().isEmpty() && !outType.getTypeParameters().get(0).isWildcard()) {
                String typeName = outType.getTypeParameters().get(0).getTypeOrClassname();
                eventType = resolveType(typeName, types, servicesContext);
            }
            else {
                isWildcard = true;
            }
        }
        else if (typeOrClassname != null) {
            eventType = resolveType(typeOrClassname, types, servicesContext);
        }
        else {
            isWildcard = true;
        }
        return new GraphTypeDesc(isWildcard, isUnderlying, eventType);
    }

    private static EventType resolveType(String typeOrClassname, Map<String, EventType> types, EPServicesContext servicesContext)
            throws ExprValidationException
    {
        EventType eventType = types.get(typeOrClassname);
        if (eventType == null) {
            eventType = servicesContext.getEventAdapterService().getExistsTypeByName(typeOrClassname);
        }
        if (eventType == null) {
            throw new ExprValidationException("Failed to find event type '" + typeOrClassname + "'");
        }
        return eventType;
    }

    private void compileTimeValidate(CreateDataFlowDesc desc) throws ExprValidationException {
        for (GraphOperatorSpec spec : desc.getOperators()) {
            for (GraphOperatorOutputItem out : spec.getOutput().getItems()) {
                if (out.getTypeInfo().size() > 1) {
                    throw new ExprValidationException("Failed to validate operator '" + spec.getOperatorName() + "': Multiple output types for a single stream '" + out.getStreamName() + "' are not supported");
                }
            }
        }

        Set<String> schemaNames = new HashSet<String>();
        for (CreateSchemaDesc schema : desc.getSchemas()) {
            if (schemaNames.contains(schema.getSchemaName())) {
                throw new ExprValidationException("Schema name '" + schema.getSchemaName() + "' is declared more then once");
            }
            schemaNames.add(schema.getSchemaName());
        }
    }
}