/*
* (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Contributors:
* Tiry
* Benoit Delbosc
*/
package org.nuxeo.elasticsearch.audit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.DateTime;
import org.joda.time.format.ISODateTimeFormat;
import org.nuxeo.common.utils.TextTemplate;
import org.nuxeo.ecm.core.api.DocumentModel;
import org.nuxeo.ecm.core.api.NuxeoException;
import org.nuxeo.ecm.core.uidgen.UIDGeneratorService;
import org.nuxeo.ecm.core.uidgen.UIDSequencer;
import org.nuxeo.ecm.core.work.api.Work;
import org.nuxeo.ecm.core.work.api.Work.State;
import org.nuxeo.ecm.core.work.api.WorkManager;
import org.nuxeo.ecm.platform.audit.api.AuditReader;
import org.nuxeo.ecm.platform.audit.api.ExtendedInfo;
import org.nuxeo.ecm.platform.audit.api.FilterMapEntry;
import org.nuxeo.ecm.platform.audit.api.LogEntry;
import org.nuxeo.ecm.platform.audit.api.query.AuditQueryException;
import org.nuxeo.ecm.platform.audit.api.query.DateRangeParser;
import org.nuxeo.ecm.platform.audit.service.AbstractAuditBackend;
import org.nuxeo.ecm.platform.audit.service.AuditBackend;
import org.nuxeo.ecm.platform.audit.service.BaseLogEntryProvider;
import org.nuxeo.ecm.platform.audit.service.NXAuditEventsService;
import org.nuxeo.ecm.platform.audit.service.extension.AuditBackendDescriptor;
import org.nuxeo.ecm.platform.query.api.PredicateDefinition;
import org.nuxeo.ecm.platform.query.api.PredicateFieldDefinition;
import org.nuxeo.elasticsearch.ElasticSearchConstants;
import org.nuxeo.elasticsearch.api.ElasticSearchAdmin;
import org.nuxeo.elasticsearch.audit.io.AuditEntryJSONReader;
import org.nuxeo.elasticsearch.audit.io.AuditEntryJSONWriter;
import org.nuxeo.runtime.api.Framework;
import org.nuxeo.runtime.model.DefaultComponent;
/**
* Implementation of the {@link AuditBackend} interface using Elasticsearch persistence
*
* @author tiry
*/
public class ESAuditBackend extends AbstractAuditBackend implements AuditBackend {
public static final String SEQ_NAME = "audit";
public static final String MIGRATION_FLAG_PROP = "audit.elasticsearch.migration";
public static final String MIGRATION_BATCH_SIZE_PROP = "audit.elasticsearch.migration.batchSize";
public static final String MIGRATION_DONE_EVENT = "sqlToElasticsearchMigrationDone";
public static final int MIGRATION_DEFAULT_BACTH_SIZE = 1000;
public ESAuditBackend(NXAuditEventsService component, AuditBackendDescriptor config) {
super(component, config);
}
protected Client esClient;
protected static final Log log = LogFactory.getLog(ESAuditBackend.class);
protected BaseLogEntryProvider provider = new BaseLogEntryProvider() {
@Override
public int removeEntries(String eventId, String pathPattern) {
throw new UnsupportedOperationException("Not implemented yet!");
}
@Override
public void addLogEntry(LogEntry logEntry) {
List<LogEntry> entries = new ArrayList<>();
entries.add(logEntry);
addLogEntries(entries);
}
@Override
public List<LogEntry> getLogEntriesFor(String uuid, String repositoryId) {
throw new UnsupportedOperationException("Not implemented yet!");
}
@Override
public List<LogEntry> getLogEntriesFor(String uuid) {
throw new UnsupportedOperationException("Not implemented yet!");
}
@Override
public List<LogEntry> getLogEntriesFor(String uuid, Map<String, FilterMapEntry> filterMap,
boolean doDefaultSort) {
throw new UnsupportedOperationException("Not implemented yet!");
}
};
protected Client getClient() {
log.info("Activate Elasticsearch backend for Audit");
ElasticSearchAdmin esa = Framework.getService(ElasticSearchAdmin.class);
Client client = esa.getClient();
ensureUIDSequencer(client);
return client;
}
protected boolean isMigrationDone() {
AuditReader reader = Framework.getService(AuditReader.class);
List<LogEntry> entries = reader.queryLogs(new String[] { MIGRATION_DONE_EVENT }, null);
return !entries.isEmpty();
}
@Override
public int getApplicationStartedOrder() {
int elasticOrder = ((DefaultComponent) Framework.getRuntime()
.getComponent("org.nuxeo.elasticsearch.ElasticSearchComponent"))
.getApplicationStartedOrder();
int uidgenOrder = ((DefaultComponent) Framework.getRuntime()
.getComponent("org.nuxeo.ecm.core.uidgen.UIDGeneratorService"))
.getApplicationStartedOrder();
return Integer.max(elasticOrder, uidgenOrder) + 1;
}
@Override
public void onApplicationStarted() {
esClient = getClient();
if (Boolean.parseBoolean(Framework.getProperty(MIGRATION_FLAG_PROP))) {
if (!isMigrationDone()) {
log.info(String.format(
"Property %s is true and migration is not done yet, processing audit migration from SQL to Elasticsearch index",
MIGRATION_FLAG_PROP));
// Drop audit index first in case of a previous bad migration
ElasticSearchAdmin esa = Framework.getService(ElasticSearchAdmin.class);
esa.dropAndInitIndex(getESIndexName());
int batchSize = MIGRATION_DEFAULT_BACTH_SIZE;
String batchSizeProp = Framework.getProperty(MIGRATION_BATCH_SIZE_PROP);
if (batchSizeProp != null) {
batchSize = Integer.parseInt(batchSizeProp);
}
migrate(batchSize);
} else {
log.warn(String.format(
"Property %s is true but migration is already done, please set this property to false",
MIGRATION_FLAG_PROP));
}
} else {
log.debug(String.format("Property %s is false, not processing any migration", MIGRATION_FLAG_PROP));
}
}
@Override
public void onApplicationStopped() {
if (esClient == null) {
return;
}
try {
esClient.close();
} finally {
esClient = null;
}
}
@Override
public List<LogEntry> getLogEntriesFor(String uuid, String repositoryId) {
TermQueryBuilder docFilter = QueryBuilders.termQuery("docUUID", uuid);
TermQueryBuilder repoFilter = QueryBuilders.termQuery("repositoryId", repositoryId);
QueryBuilder filter;
filter = QueryBuilders.boolQuery().must(docFilter);
filter = QueryBuilders.boolQuery().must(repoFilter);
return getLogEntries(filter, false);
}
@Override
public List<LogEntry> getLogEntriesFor(String uuid, Map<String, FilterMapEntry> filterMap, boolean doDefaultSort) {
TermQueryBuilder docFilter = QueryBuilders.termQuery("docUUID", uuid);
QueryBuilder filter;
if (MapUtils.isEmpty(filterMap)) {
filter = docFilter;
} else {
filter = QueryBuilders.boolQuery().must(docFilter);
for (String key : filterMap.keySet()) {
FilterMapEntry entry = filterMap.get(key);
((BoolQueryBuilder) filter).must(QueryBuilders.termQuery(entry.getColumnName(), entry.getObject()));
}
}
return getLogEntries(filter, doDefaultSort);
}
protected List<LogEntry> getLogEntries(QueryBuilder filter, boolean doDefaultSort) {
SearchRequestBuilder builder = getSearchRequestBuilder(esClient);
if (doDefaultSort) {
builder.addSort("eventDate", SortOrder.DESC);
}
TimeValue keepAlive = TimeValue.timeValueMinutes(1);
builder.setQuery(QueryBuilders.constantScoreQuery(filter)).setScroll(keepAlive).setSize(100);
logSearchRequest(builder);
SearchResponse searchResponse = builder.get();
logSearchResponse(searchResponse);
// Build log entries
List<LogEntry> logEntries = buildLogEntries(searchResponse);
// Scroll on next results
for (; //
searchResponse.getHits().getHits().length > 0
&& logEntries.size() < searchResponse.getHits().getTotalHits(); //
searchResponse = runNextScroll(searchResponse.getScrollId(), keepAlive)) {
// Build log entries
logEntries.addAll(buildLogEntries(searchResponse));
}
return logEntries;
}
SearchResponse runNextScroll(String scrollId, TimeValue keepAlive) {
if (log.isDebugEnabled()) {
log.debug(String.format(
"Scroll request: -XGET 'localhost:9200/_search/scroll' -d '{\"scroll\": \"%s\", \"scroll_id\": \"%s\" }'",
keepAlive, scrollId));
}
SearchResponse response = esClient.prepareSearchScroll(scrollId).setScroll(keepAlive).execute().actionGet();
logSearchResponse(response);
return response;
}
protected List<LogEntry> buildLogEntries(SearchResponse searchResponse) {
List<LogEntry> entries = new ArrayList<>(searchResponse.getHits().getHits().length);
for (SearchHit hit : searchResponse.getHits()) {
try {
entries.add(AuditEntryJSONReader.read(hit.getSourceAsString()));
} catch (IOException e) {
log.error("Error while reading Audit Entry from ES", e);
}
}
return entries;
}
protected SearchRequestBuilder getSearchRequestBuilder(Client esClient) {
return esClient.prepareSearch(getESIndexName())
.setTypes(ElasticSearchConstants.ENTRY_TYPE)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
}
@Override
public LogEntry getLogEntryByID(long id) {
GetResponse ret = esClient.prepareGet(getESIndexName(), ElasticSearchConstants.ENTRY_TYPE, String.valueOf(id))
.get();
if (!ret.isExists()) {
return null;
}
try {
return AuditEntryJSONReader.read(ret.getSourceAsString());
} catch (IOException e) {
throw new RuntimeException("Unable to read Entry for id " + id, e);
}
}
public SearchRequestBuilder buildQuery(String query, Map<String, Object> params) {
if (params != null && params.size() > 0) {
query = expandQueryVariables(query, params);
}
SearchRequestBuilder builder = getSearchRequestBuilder(esClient);
builder.setQuery(query);
return builder;
}
public String expandQueryVariables(String query, Object[] params) {
Map<String, Object> qParams = new HashMap<>();
for (int i = 0; i < params.length; i++) {
query = query.replaceFirst("\\?", "\\${param" + i + "}");
qParams.put("param" + i, params[i]);
}
return expandQueryVariables(query, qParams);
}
public String expandQueryVariables(String query, Map<String, Object> params) {
if (params != null && params.size() > 0) {
TextTemplate tmpl = new TextTemplate();
for (String key : params.keySet()) {
Object val = params.get(key);
if (val == null) {
continue;
} else if (val instanceof Calendar) {
tmpl.setVariable(key, ISODateTimeFormat.dateTime().print(new DateTime(val)));
} else if (val instanceof Date) {
tmpl.setVariable(key, ISODateTimeFormat.dateTime().print(new DateTime(val)));
} else {
tmpl.setVariable(key, val.toString());
}
}
query = tmpl.processText(query);
}
return query;
}
@Override
public List<?> nativeQuery(String query, Map<String, Object> params, int pageNb, int pageSize) {
SearchRequestBuilder builder = buildQuery(query, params);
if (pageNb > 0) {
builder.setFrom(pageNb * pageSize);
}
if (pageSize > 0) {
builder.setSize(pageSize);
}
logSearchRequest(builder);
SearchResponse searchResponse = builder.get();
logSearchResponse(searchResponse);
return buildLogEntries(searchResponse);
}
@Override
public List<LogEntry> queryLogsByPage(String[] eventIds, Date limit, String[] categories, String path, int pageNb,
int pageSize) {
SearchRequestBuilder builder = getSearchRequestBuilder(esClient);
BoolQueryBuilder filterBuilder = QueryBuilders.boolQuery();
if (eventIds != null && eventIds.length > 0) {
if (eventIds.length == 1) {
filterBuilder.must(QueryBuilders.termQuery("eventId", eventIds[0]));
} else {
filterBuilder.must(QueryBuilders.termsQuery("eventId", eventIds));
}
}
if (categories != null && categories.length > 0) {
if (categories.length == 1) {
filterBuilder.must(QueryBuilders.termQuery("category", categories[0]));
} else {
filterBuilder.must(QueryBuilders.termsQuery("category", categories));
}
}
if (path != null) {
filterBuilder.must(QueryBuilders.termQuery("docPath", path));
}
if (limit != null) {
filterBuilder.must(QueryBuilders.rangeQuery("eventDate").lt(limit));
}
builder.setQuery(QueryBuilders.constantScoreQuery(filterBuilder));
if (pageNb > 0) {
builder.setFrom(pageNb * pageSize);
}
if (pageSize > 0) {
builder.setSize(pageSize);
}
logSearchRequest(builder);
SearchResponse searchResponse = builder.get();
logSearchResponse(searchResponse);
return buildLogEntries(searchResponse);
}
@Override
public List<LogEntry> queryLogsByPage(String[] eventIds, String dateRange, String[] categories, String path,
int pageNb, int pageSize) {
Date limit = null;
if (dateRange != null) {
try {
limit = DateRangeParser.parseDateRangeQuery(new Date(), dateRange);
} catch (AuditQueryException aqe) {
aqe.addInfo("Wrong date range query. Query was " + dateRange);
throw aqe;
}
}
return queryLogsByPage(eventIds, limit, categories, path, pageNb, pageSize);
}
@Override
public void addLogEntries(List<LogEntry> entries) {
if (entries.isEmpty()) {
return;
}
BulkRequestBuilder bulkRequest = esClient.prepareBulk();
JsonFactory factory = new JsonFactory();
UIDGeneratorService uidGeneratorService = Framework.getService(UIDGeneratorService.class);
UIDSequencer seq = uidGeneratorService.getSequencer();
try {
for (LogEntry entry : entries) {
entry.setId(seq.getNext(SEQ_NAME));
if (log.isDebugEnabled()) {
log.debug(String.format("Indexing log entry: %s", entry));
}
OutputStream out = new BytesStreamOutput();
JsonGenerator jsonGen = factory.createJsonGenerator(out);
XContentBuilder builder = jsonBuilder(out);
AuditEntryJSONWriter.asJSON(jsonGen, entry);
bulkRequest.add(esClient.prepareIndex(getESIndexName(), ElasticSearchConstants.ENTRY_TYPE,
String.valueOf(entry.getId())).setSource(builder));
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
for (BulkItemResponse response : bulkResponse.getItems()) {
if (response.isFailed()) {
log.error("Unable to index audit entry " + response.getItemId() + " :"
+ response.getFailureMessage());
}
}
}
} catch (IOException e) {
throw new NuxeoException("Error while indexing Audit entries", e);
}
}
@Override
public Long getEventsCount(String eventId) {
SearchResponse res = esClient.prepareSearch(getESIndexName())
.setTypes(ElasticSearchConstants.ENTRY_TYPE)
.setQuery(QueryBuilders.constantScoreQuery(
QueryBuilders.termQuery("eventId", eventId)))
.setSize(0)
.get();
return res.getHits().getTotalHits();
}
@Override
public long syncLogCreationEntries(final String repoId, final String path, final Boolean recurs) {
return syncLogCreationEntries(provider, repoId, path, recurs);
}
protected QueryBuilder buildFilter(PredicateDefinition[] predicates, DocumentModel searchDocumentModel) {
if (searchDocumentModel == null) {
return QueryBuilders.matchAllQuery();
}
BoolQueryBuilder filterBuilder = QueryBuilders.boolQuery();
int nbFilters = 0;
for (PredicateDefinition predicate : predicates) {
// extract data from DocumentModel
PredicateFieldDefinition[] fieldDef = predicate.getValues();
Object[] val = new Object[fieldDef.length];
for (int fidx = 0; fidx < fieldDef.length; fidx++) {
if (fieldDef[fidx].getXpath() != null) {
val[fidx] = searchDocumentModel.getPropertyValue(fieldDef[fidx].getXpath());
} else {
val[fidx] = searchDocumentModel.getProperty(fieldDef[fidx].getSchema(), fieldDef[fidx].getName());
}
}
if (!isNonNullParam(val)) {
// skip predicate where all values are null
continue;
}
nbFilters++;
String op = predicate.getOperator();
if (op.equalsIgnoreCase("IN")) {
String[] values = null;
if (val[0] instanceof Iterable<?>) {
List<String> l = new ArrayList<>();
Iterable<?> vals = (Iterable<?>) val[0];
for (Object v : vals) {
if (v != null) {
l.add(v.toString());
}
}
values = l.toArray(new String[l.size()]);
} else if (val[0] instanceof Object[]) {
values = (String[]) val[0];
}
filterBuilder.must(QueryBuilders.termsQuery(predicate.getParameter(), values));
} else if (op.equalsIgnoreCase("BETWEEN")) {
filterBuilder.must(QueryBuilders.rangeQuery(predicate.getParameter()).gt(val[0]));
if (val.length > 1) {
filterBuilder.must(QueryBuilders.rangeQuery(predicate.getParameter()).lt(val[1]));
}
} else if (">".equals(op)) {
filterBuilder.must(QueryBuilders.rangeQuery(predicate.getParameter()).gt(val[0]));
} else if (">=".equals(op)) {
filterBuilder.must(QueryBuilders.rangeQuery(predicate.getParameter()).gte(val[0]));
} else if ("<".equals(op)) {
filterBuilder.must(QueryBuilders.rangeQuery(predicate.getParameter()).lt(val[0]));
} else if ("<=".equals(op)) {
filterBuilder.must(QueryBuilders.rangeQuery(predicate.getParameter()).lte(val[0]));
} else {
filterBuilder.must(QueryBuilders.termQuery(predicate.getParameter(), val[0]));
}
}
if (nbFilters == 0) {
return QueryBuilders.matchAllQuery();
}
return filterBuilder;
}
public SearchRequestBuilder buildSearchQuery(String fixedPart, PredicateDefinition[] predicates,
DocumentModel searchDocumentModel) {
SearchRequestBuilder builder = getSearchRequestBuilder(esClient);
QueryBuilder queryBuilder = QueryBuilders.wrapperQuery(fixedPart);
QueryBuilder filterBuilder = buildFilter(predicates, searchDocumentModel);
builder.setQuery(QueryBuilders.boolQuery().must(queryBuilder).filter(filterBuilder));
return builder;
}
protected boolean isNonNullParam(Object[] val) {
if (val == null) {
return false;
}
for (Object v : val) {
if (v != null) {
if (v instanceof String) {
if (!((String) v).isEmpty()) {
return true;
}
} else if (v instanceof String[]) {
if (((String[]) v).length > 0) {
return true;
}
} else {
return true;
}
}
}
return false;
}
@SuppressWarnings("deprecation")
public String migrate(final int batchSize) {
final String MIGRATION_WORK_ID = "AuditMigration";
WorkManager wm = Framework.getService(WorkManager.class);
State migrationState = wm.getWorkState(MIGRATION_WORK_ID);
if (migrationState != null) {
return "Migration already scheduled : " + migrationState.toString();
}
Work migrationWork = new ESAuditMigrationWork(MIGRATION_WORK_ID, batchSize);
wm.schedule(migrationWork);
return "Migration work started : " + MIGRATION_WORK_ID;
}
protected void logSearchResponse(SearchResponse response) {
if (log.isDebugEnabled()) {
log.debug("Response: " + response.toString());
}
}
protected void logSearchRequest(SearchRequestBuilder request) {
if (log.isDebugEnabled()) {
log.debug(String.format("Search query: curl -XGET 'http://localhost:9200/%s/%s/_search?pretty' -d '%s'",
getESIndexName(), ElasticSearchConstants.ENTRY_TYPE, request.toString()));
}
}
/**
* Ensures the audit sequence returns an UID greater or equal than the maximum log entry id.
*/
protected void ensureUIDSequencer(Client esClient) {
boolean auditIndexExists = esClient.admin()
.indices()
.prepareExists(getESIndexName())
.execute()
.actionGet()
.isExists();
if (!auditIndexExists) {
return;
}
// Get max log entry id
SearchRequestBuilder builder = getSearchRequestBuilder(esClient);
builder.setQuery(QueryBuilders.matchAllQuery()).addAggregation(AggregationBuilders.max("maxAgg").field("id"));
SearchResponse searchResponse = builder.execute().actionGet();
Max agg = searchResponse.getAggregations().get("maxAgg");
int maxLogEntryId = (int) agg.getValue();
// Get next sequence id
UIDGeneratorService uidGeneratorService = Framework.getService(UIDGeneratorService.class);
UIDSequencer seq = uidGeneratorService.getSequencer();
seq.init();
int nextSequenceId = seq.getNext(SEQ_NAME);
// Increment sequence to max log entry id if needed
if (nextSequenceId < maxLogEntryId) {
log.info(String.format("Next UID returned by %s sequence is %d, initializing sequence to %d", SEQ_NAME,
nextSequenceId, maxLogEntryId));
seq.initSequence(SEQ_NAME, maxLogEntryId);
}
}
@Override
public ExtendedInfo newExtendedInfo(Serializable value) {
return new ESExtendedInfo(value);
}
protected String getESIndexName() {
ElasticSearchAdmin esa = Framework.getService(ElasticSearchAdmin.class);
return esa.getIndexNameForType(ElasticSearchConstants.ENTRY_TYPE);
}
}