/*
* Copyright (C) 2015 Orange
*
* This software is distributed under the terms and conditions of the 'GNU GENERAL PUBLIC LICENSE
* Version 2' license which can be found in the file 'LICENSE.txt' in this package distribution or
* at 'http://www.gnu.org/licenses/gpl-2.0-standalone.html'.
*/
package com.orange.cepheus.cep;
import com.espertech.esper.client.*;
import com.espertech.esper.client.soda.EPStatementObjectModel;
import com.orange.cepheus.cep.exception.ConfigurationException;
import com.orange.cepheus.cep.exception.EventProcessingException;
import com.orange.cepheus.cep.exception.EventTypeNotFoundException;
import com.orange.cepheus.cep.model.*;
import com.orange.cepheus.cep.model.Configuration;
import com.orange.cepheus.cep.model.EventType;
import com.orange.cepheus.cep.tenant.TenantScope;
import com.orange.cepheus.geo.Geospatial;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.actuate.metrics.GaugeService;
import javax.annotation.PostConstruct;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
/**
* ComplexEventProcessor implementation using EsperTech Esper CEP
*/
public class EsperEventProcessor implements ComplexEventProcessor {
private static Logger logger = LoggerFactory.getLogger(EsperEventProcessor.class);
private EPServiceProvider epServiceProvider;
private Configuration configuration;
/**
* Keep a list of statements that declare a variable (create variable),
* required to properly remove them on updates as Esper does not provide this.
*/
private HashMap<String, String> variablesByStatementName = new HashMap<>();
/**
* Collect Esper metrics when the Spring Boot metrics are enabled
*/
@Value("${endpoints.metrics.enabled:${endpoints.enabled:false}}")
private boolean collectMetrics;
/**
* This bean is only injected in multi tenant mode.
*/
@Autowired(required = false)
private TenantScope tenantScope;
@Autowired
private EventMapper eventMapper;
@Autowired
private EventSinkListener eventSinkListener;
/**
* Expose Esper statement metrics as Spring Boot metrics
*/
@Autowired
private GaugeService gaugeService;
@PostConstruct
public void init() {
com.espertech.esper.client.Configuration configuration = new com.espertech.esper.client.Configuration();
Geospatial.registerConfiguration(configuration);
if (collectMetrics) {
logger.warn("Activating Esper metrics, expect performance impacts");
configuration.setMetricsReportingEnabled();
}
if (tenantScope != null) {
String provider = tenantScope.getConversationId();
epServiceProvider = EPServiceProviderManager.getProvider(provider, configuration);
} else {
epServiceProvider = EPServiceProviderManager.getDefaultProvider(configuration);
}
}
/**
* Apply a new configuration to the Esper CEP.
* @param configuration the new configuration to apply
*/
public void setConfiguration(Configuration configuration) throws ConfigurationException {
logger.info("Apply configuration");
Configuration previousConfiguration = this.configuration;
ConfigurationOperations operations = epServiceProvider.getEPAdministrator().getConfiguration();
try {
Collection<EventType> previousEventTypes = Collections.emptyList();
// Update incoming event types
Collection<EventType> newEventTypes = Collections.unmodifiableList(configuration.getEventTypeIns());
if (previousConfiguration != null) {
previousEventTypes = Collections.unmodifiableList(previousConfiguration.getEventTypeIns());
}
this.updateEventTypes(previousEventTypes, newEventTypes, operations);
// Update outgoing event types
newEventTypes = Collections.unmodifiableList(configuration.getEventTypeOuts());
if (previousConfiguration != null) {
previousEventTypes = Collections.unmodifiableList(previousConfiguration.getEventTypeOuts());
}
this.updateEventTypes(previousEventTypes, newEventTypes, operations);
// Update the statements
this.updateStatements(configuration.getStatements());
this.configuration = configuration;
eventSinkListener.setConfiguration(configuration);
} catch (Exception e) {
throw new ConfigurationException("Failed to apply new configuration", e);
}
}
public Configuration getConfiguration() {
return configuration;
}
/**
* Restores the active configuration by wiping out the complete set of active statements and event types.
* This operation will lock the entire CEP engine.
* @return true if the restoration was successful, false if the CEP failed to reinitialize from the active configuration
*/
public boolean restoreConfiguration(Configuration previousConfiguration) {
epServiceProvider.getEngineInstanceWideLock().writeLock().lock();
try {
ConfigurationOperations operations = epServiceProvider.getEPAdministrator().getConfiguration();
// Cleanup previous configuration
epServiceProvider.getEPAdministrator().destroyAllStatements();
for (com.espertech.esper.client.EventType eventType : operations.getEventTypes()) {
operations.removeEventType(eventType.getName(), true);
}
// Remove all statements to variables associations
variablesByStatementName.clear();
// Adding back in/out events, then statements
Collection<EventType> inEventTypes = Collections.unmodifiableList(previousConfiguration.getEventTypeIns());
Collection<EventType> outEventTypes = Collections.unmodifiableList(previousConfiguration.getEventTypeOuts());
this.updateEventTypes(Collections.emptyList(), inEventTypes, operations);
this.updateEventTypes(Collections.emptyList(), outEventTypes, operations);
this.updateStatements(previousConfiguration.getStatements());
} catch (Exception e) {
logger.error("Failed to restore active configuration", e);
this.configuration = null;
return false;
} finally {
epServiceProvider.getEngineInstanceWideLock().writeLock().unlock();
}
return true;
}
/**
* Reset the CEP and remove the configuration
*/
public void reset() {
epServiceProvider.destroy();
epServiceProvider.initialize();
configuration = null;
variablesByStatementName.clear();
}
/**
* Make Esper process an event
* @param event
* @throws EventProcessingException
*/
public void processEvent(Event event) throws EventProcessingException {
logger.info("EventIn: {}", event.toString());
try {
this.epServiceProvider.getEPRuntime().sendEvent(event.getValues(), event.getType());
} catch (EPException|EPServiceDestroyedException e) {
throw new EventProcessingException(e.getMessage());
}
}
/**
* Return a list of Attribute for a given even type. This is mainly useful for testing.
* @param eventTypeName
* @return
* @throws EventTypeNotFoundException
*/
public Map<String, Attribute> getEventTypeAttributes(String eventTypeName) throws EventTypeNotFoundException {
Map<String, Attribute> attributes = new HashMap<>();
com.espertech.esper.client.EventType eventType = epServiceProvider.getEPAdministrator().getConfiguration().getEventType(eventTypeName);
if (eventType != null){
for (String name : eventType.getPropertyNames()) {
if (!("id".equals(name))) {
String type = eventType.getPropertyType(name).getSimpleName().toLowerCase();
attributes.put(name, new Attribute(name, type));
}
}
} else {
throw new EventTypeNotFoundException("The event type does not exist.");
}
return attributes;
}
/**
* Return the list of EPL statements.
* @return a list of EPL statements
*/
public List<Statement> getStatements() {
List<Statement> statements = new LinkedList<>();
for (String statementName : epServiceProvider.getEPAdministrator().getStatementNames()) {
EPStatement epStatement = epServiceProvider.getEPAdministrator().getStatement(statementName);
if (epStatement != null) {
Statement statement = new Statement(epStatement.getName(), epStatement.getText());
statements.add(statement);
}
}
return statements;
}
/**
* Return the list of EPL statements. This is mainly useful for testing.
* @return a list of EPL statements
*/
public List<EPStatement> getEPStatements() {
List<EPStatement> statements = new LinkedList<>();
for (String statementName : epServiceProvider.getEPAdministrator().getStatementNames()) {
EPStatement statement = epServiceProvider.getEPAdministrator().getStatement(statementName);
if (statement != null) {
statements.add(statement);
}
}
return statements;
}
/**
* Update the CEP event types by adding new types and removing the older ones.
*
* @param oldList the previous list of event types
* @param newList the new list of event types
* @param operations the CEP configuration
*/
private void updateEventTypes(Collection<EventType> oldList, Collection<EventType> newList, ConfigurationOperations operations) {
List<EventType> eventTypesToRemove = new LinkedList<>(oldList);
eventTypesToRemove.removeAll(newList);
List<EventType> eventTypesToAdd = new LinkedList<>(newList);
eventTypesToAdd.removeAll(oldList);
// List all statements depending on the event types to remove
Set<String> statementsToDelete = new HashSet<>();
for (EventType eventType : eventTypesToRemove) {
statementsToDelete.addAll(operations.getEventTypeNameUsedBy(eventType.getType()));
}
// Delete all the statements depending on the event types to remove
for (String statementName : statementsToDelete) {
EPStatement statement = epServiceProvider.getEPAdministrator().getStatement(statementName);
if (statement != null) {
logger.info("Removing unused statement: "+statement.getText());
statement.stop();
statement.destroy();
}
}
// Finally remove the event types
for (EventType eventType : eventTypesToRemove) {
logger.info("Removing unused event: " + eventType);
operations.removeEventType(eventType.getType(), false);
}
for (EventType eventType : eventTypesToAdd) {
logger.info("Add new event type: {}", eventType);
// Add event type mapped to esper representation
String eventTypeName = eventType.getType();
operations.addEventType(eventTypeName, eventMapper.esperTypeFromEventType(eventType));
}
}
/**
* Update the EPL statements by adding new statements, and removing unused statements
* @param statements
* @throws NoSuchAlgorithmException
*/
private void updateStatements(Collection<String> statements) throws NoSuchAlgorithmException {
// Keep a list of MD5 hash of all added statements
Set<String> hashes = new HashSet<>();
for (String eplStatement : statements) {
hashes.add(MD5(eplStatement));
}
// Removed unused statements first
for (String hash : epServiceProvider.getEPAdministrator().getStatementNames()) {
if (!hashes.contains(hash)) {
removeStatement(hash);
}
}
// Update EPL statements
for (String eplStatement : statements) {
String hash = MD5(eplStatement);
// Create statement if does not already exist
EPStatement statement = epServiceProvider.getEPAdministrator().getStatement(hash);
if (statement == null) {
logger.info("Add new statement: {}", eplStatement);
EPStatementObjectModel model = epServiceProvider.getEPAdministrator().compileEPL(eplStatement);
// When the statement defines a new variable, keep track of this association
if (model.getCreateVariable() != null) {
variablesByStatementName.put(hash, model.getCreateVariable().getVariableName());
}
statement = epServiceProvider.getEPAdministrator().create(model, hash);
statement.addListener(eventSinkListener);
}
}
// Collect metrics statements if enabled
if (collectMetrics) {
EPStatement statement = epServiceProvider.getEPAdministrator().createEPL("select * from com.espertech.esper.client.metric.StatementMetric", "STATEMENT_METRIC");
statement.addListener((eventBeans, unused) -> {
if (eventBeans != null) {
for (EventBean eventBean : eventBeans) {
String statementName = (String)eventBean.get("statementName");
String tenant = (String)eventBean.get("engineURI");
String keyPrefix = "cepheus.statement."+tenant+"."+statementName;
gaugeService.submit(keyPrefix+".cpuTime", (Long)eventBean.get("cpuTime"));
gaugeService.submit(keyPrefix+".wallTime", (Long)eventBean.get("wallTime"));
gaugeService.submit(keyPrefix+".numInput", (Long)eventBean.get("numInput"));
gaugeService.submit(keyPrefix+".numOutputIStream", (Long)eventBean.get("numOutputIStream"));
}
}
});
}
}
/**
* Generate the MD5 hash of a message
* @param message
* @return the hash
* @throws NoSuchAlgorithmException
*/
private String MD5(String message) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] array = md.digest(message.getBytes());
return new BigInteger(1, array).toString(16);
/*StringBuffer sb = new StringBuffer();
for (int i = 0; i < array.length; ++i) {
sb.append(Integer.toHexString((array[i] & 0xFF) | 0x100).substring(1,3));
}
return sb.toString();*/
}
/**
* Remove a statement, recursively removing statements depending on it
* when it is a statement declaring a variable
* @param statementName the hash of the statement to delete
*/
private void removeStatement(String statementName) {
EPStatement statement = epServiceProvider.getEPAdministrator().getStatement(statementName);
if (statement != null) {
logger.info("Remove statement: {}", statement.getText());
// Destroy all statements associated to a given statement declaring a variable
String variableName = variablesByStatementName.get(statementName);
if (variableName != null) {
Set<String> epStatements = epServiceProvider.getEPAdministrator().getConfiguration().getVariableNameUsedBy(variableName);
for (String epStatement : new LinkedList<>(epStatements)) { // use a new list to prevent conccurent access
if (!epStatement.equals(statementName)) {
removeStatement(epStatement);
}
}
}
//TODO support other type of statement dependency ?
statement.destroy();
}
}
}