/**
* Copyright (c) Codice Foundation
*
* This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser
* General Public License as published by the Free Software Foundation, either version 3 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details. A copy of the GNU Lesser General Public License
* is distributed along with this program and can be found at
* <http://www.gnu.org/licenses/lgpl.html>.
*
**/
package org.codice.ddf.ui.searchui.query.controller;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.collections.map.LRUMap;
import org.apache.commons.lang.StringUtils;
import org.codice.ddf.ui.searchui.query.model.QueryStatus;
import org.codice.ddf.ui.searchui.query.model.Search;
import org.codice.ddf.ui.searchui.query.model.SearchRequest;
import org.cometd.bayeux.server.BayeuxServer;
import org.cometd.bayeux.server.ConfigurableServerChannel;
import org.cometd.bayeux.server.ServerMessage;
import org.cometd.bayeux.server.ServerSession;
import org.cometd.server.ServerMessageImpl;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.opengis.filter.expression.PropertyName;
import org.opengis.filter.sort.SortBy;
import org.opengis.filter.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Ordering;
import ddf.action.Action;
import ddf.action.ActionRegistry;
import ddf.catalog.CatalogFramework;
import ddf.catalog.data.Attribute;
import ddf.catalog.data.AttributeDescriptor;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.MetacardType;
import ddf.catalog.data.Result;
import ddf.catalog.federation.FederationException;
import ddf.catalog.operation.Query;
import ddf.catalog.operation.QueryRequest;
import ddf.catalog.operation.QueryResponse;
import ddf.catalog.operation.SourceResponse;
import ddf.catalog.operation.impl.ProcessingDetailsImpl;
import ddf.catalog.operation.impl.QueryRequestImpl;
import ddf.catalog.operation.impl.QueryResponseImpl;
import ddf.catalog.source.SourceUnavailableException;
import ddf.catalog.source.UnsupportedQueryException;
import ddf.catalog.transform.CatalogTransformerException;
import ddf.catalog.transformer.metacard.geojson.GeoJsonMetacardTransformer;
import ddf.catalog.util.impl.DistanceResultComparator;
import ddf.catalog.util.impl.RelevanceResultComparator;
import ddf.catalog.util.impl.TemporalResultComparator;
import ddf.security.SecurityConstants;
import ddf.security.Subject;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
/**
* The SearchController class handles all of the query threads for asynchronous queries.
*/
public class SearchController {
private static final Logger LOGGER = LoggerFactory.getLogger(SearchController.class);
private static final DateTimeFormatter ISO_8601_DATE_FORMAT = DateTimeFormat
.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").withZoneUTC();
@SuppressWarnings("serial")
private static final Map<String, Serializable> INDEX_PROPERTIES = Collections
.unmodifiableMap(new HashMap<String, Serializable>() {
{
put("mode", "index");
}
});
@SuppressWarnings("serial")
private static final Map<String, Serializable> CACHE_PROPERTIES = Collections
.unmodifiableMap(new HashMap<String, Serializable>() {
{
put("mode", "cache");
}
});
private final ExecutorService executorService = getExecutorService();
// TODO: just store the searches in memory for now, change this later
private final Map<String, Search> searchMap = Collections.synchronizedMap(new LRUMap(1000));
private Boolean cacheDisabled = false;
private CatalogFramework framework;
private ActionRegistry actionRegistry;
private BayeuxServer bayeuxServer;
/**
* Create a new SearchController
*
* @param framework
* - CatalogFramework that will be handling the actual queries
*/
public SearchController(CatalogFramework framework, ActionRegistry actionRegistry) {
this.framework = framework;
this.actionRegistry = actionRegistry;
}
private static void addObject(JSONObject obj, String name, Object value) {
if (value instanceof Number) {
if (value instanceof Double) {
if (((Double) value).isInfinite()) {
obj.put(name, null);
} else {
obj.put(name, value);
}
} else if (value instanceof Float) {
if (((Float) value).isInfinite()) {
obj.put(name, null);
} else {
obj.put(name, value);
}
} else {
obj.put(name, value);
}
} else if (value != null) {
obj.put(name, value);
}
}
/**
* Destroys this controller. This controller may not be used again after this method is called.
*/
public void destroy() {
executorService.shutdown();
}
/**
* Push results out to clients
* @param channel - Channel to send results on
* @param jsonData
* @param serverSession
*/
public synchronized void pushResults(String channel, JSONObject jsonData,
ServerSession serverSession) {
String channelName;
//you can't have 2 leading slashes, but if there isn't one, add it
if (channel.startsWith("/")) {
channelName = channel;
} else {
channelName = "/" + channel;
}
LOGGER.debug("Creating channel if it doesn't exist: {}", channelName);
bayeuxServer
.createChannelIfAbsent(channelName, new ConfigurableServerChannel.Initializer() {
public void configureChannel(ConfigurableServerChannel channel) {
channel.setPersistent(true);
}
});
ServerMessage.Mutable reply = new ServerMessageImpl();
reply.put(Search.SUCCESSFUL, true);
reply.putAll(jsonData);
LOGGER.debug("Sending results to subscribers on: {}", channelName);
bayeuxServer.getChannel(channelName).publish(serverSession, reply, null);
}
/**
* Execute all of the queries contained within the SearchRequest
*
* @param request
* - SearchRequest containing a query for 1 or more sources
* @param session
* - Cometd ServerSession
*/
public void executeQuery(final SearchRequest request, final ServerSession session,
final Subject subject) {
final SearchController controller = this;
if (!cacheDisabled) {
executorService.submit(new Runnable() {
@Override
public void run() {
// check if there are any currently cached results
// search cache for all sources
QueryResponse response = executeQuery(null, request, subject,
new HashMap<>(CACHE_PROPERTIES));
try {
Search search = addQueryResponseToSearch(request, response);
pushResults(request.getId(), controller.transform(search, request),
session);
} catch (InterruptedException e) {
LOGGER.error("Failed adding cached search results.", e);
} catch (CatalogTransformerException e) {
LOGGER.error("Failed to transform cached search results.", e);
}
}
});
for (final String sourceId : request.getSourceIds()) {
LOGGER.debug("Executing async query on: {}", sourceId);
executorService.submit(new Runnable() {
@Override
public void run() {
// update index from federated sources
QueryResponse indexResponse = executeQuery(sourceId, request, subject,
new HashMap<>(INDEX_PROPERTIES));
// query updated cache
QueryResponse cachedResponse = executeQuery(null, request, subject,
new HashMap<>(CACHE_PROPERTIES));
try {
Search search = addQueryResponseToSearch(request, cachedResponse);
search.updateStatus(sourceId, indexResponse);
pushResults(request.getId(), controller.transform(search, request),
session);
if (search.isFinished()) {
searchMap.remove(request.getId());
}
} catch (InterruptedException e) {
LOGGER.error("Failed adding federated search results.", e);
} catch (CatalogTransformerException e) {
LOGGER.error("Failed to transform federated search results.", e);
}
}
});
}
} else {
final Comparator<Result> sortComparator = getResultComparator(request.getQuery());
final int maxResults = request.getQuery().getPageSize() > 0 ?
request.getQuery().getPageSize() :
Integer.MAX_VALUE;
final List<Result> results = Collections.synchronizedList(new ArrayList<Result>());
for (final String sourceId : request.getSourceIds()) {
LOGGER.debug("Executing async query without cache on: {}", sourceId);
executorService.submit(new Runnable() {
@Override
public void run() {
QueryResponse sourceResponse = executeQuery(sourceId, request, subject,
new HashMap<String, Serializable>());
results.addAll(sourceResponse.getResults());
List<Result> sortedResults = Ordering.from(sortComparator)
.immutableSortedCopy(results);
sourceResponse.getResults().clear();
sourceResponse.getResults().addAll(sortedResults.size() > maxResults ?
sortedResults.subList(0, maxResults) :
sortedResults);
try {
Search search = addQueryResponseToSearch(request, sourceResponse);
search.updateStatus(sourceId, sourceResponse);
pushResults(request.getId(), controller.transform(search, request),
session);
if (search.isFinished()) {
searchMap.remove(request.getId());
}
} catch (InterruptedException e) {
LOGGER.error("Failed adding federated search results.", e);
} catch (CatalogTransformerException e) {
LOGGER.error("Failed to transform federated search results.", e);
}
}
});
}
}
}
private Comparator<Result> getResultComparator(Query query) {
Comparator<Result> sortComparator = new RelevanceResultComparator(SortOrder.DESCENDING);
SortBy sortBy = query.getSortBy();
if (sortBy != null && sortBy.getPropertyName() != null) {
PropertyName sortingProp = sortBy.getPropertyName();
String sortType = sortingProp.getPropertyName();
SortOrder sortOrder = (sortBy.getSortOrder() == null) ?
SortOrder.DESCENDING :
sortBy.getSortOrder();
// Temporal searches are currently sorted by the effective time
if (Metacard.EFFECTIVE.equals(sortType) || Result.TEMPORAL.equals(sortType)) {
sortComparator = new TemporalResultComparator(sortOrder);
} else if (Metacard.CREATED.equals(sortType) || Metacard.MODIFIED.equals(sortType)) {
sortComparator = new TemporalResultComparator(sortOrder, sortType);
} else if (Result.DISTANCE.equals(sortType)) {
sortComparator = new DistanceResultComparator(sortOrder);
} else if (Result.RELEVANCE.equals(sortType)) {
sortComparator = new RelevanceResultComparator(sortOrder);
}
}
return sortComparator;
}
private Search addQueryResponseToSearch(SearchRequest searchRequest,
QueryResponse queryResponse) throws InterruptedException {
Search search = null;
if (searchMap.containsKey(searchRequest.getId())) {
LOGGER.debug("Using previously created Search object for cache: {}",
searchRequest.getId());
search = searchMap.get(searchRequest.getId());
search.addQueryResponse(queryResponse);
} else {
LOGGER.debug("Creating new Search object to cache async query results: {}",
searchRequest.getId());
search = new Search();
search.setSearchRequest(searchRequest);
search.addQueryResponse(queryResponse);
searchMap.put(searchRequest.getId(), search);
}
return search;
}
/**
* Executes the OpenSearchQuery and formulates the response
*
* @param subject
* -the user subject
*
* @return the response on the query
*/
private QueryResponse executeQuery(String sourceId, SearchRequest searchRequest,
Subject subject, Map<String, Serializable> properties) {
Query query = searchRequest.getQuery();
QueryResponse response = getEmptyResponse(sourceId);
long startTime = System.currentTimeMillis();
try {
if (query != null) {
List<String> sourceIds;
if (sourceId == null) {
sourceIds = new ArrayList<>(searchRequest.getSourceIds());
} else {
sourceIds = Collections.singletonList(sourceId);
}
QueryRequest request = new QueryRequestImpl(query, false, sourceIds, properties);
if (subject != null) {
LOGGER.debug("Adding {} property with value {} to request.",
SecurityConstants.SECURITY_SUBJECT, subject);
request.getProperties().put(SecurityConstants.SECURITY_SUBJECT, subject);
}
LOGGER.debug("Sending query: {}", query);
response = framework.query(request);
}
} catch (UnsupportedQueryException | FederationException e) {
LOGGER.warn("Error executing query", e);
response.getProcessingDetails().add(new ProcessingDetailsImpl(sourceId, e));
} catch (SourceUnavailableException e) {
LOGGER.warn("Error executing query because the underlying source was unavailable.", e);
response.getProcessingDetails().add(new ProcessingDetailsImpl(sourceId, e));
} catch (RuntimeException e) {
// Account for any runtime exceptions and send back a server error
// this prevents full stacktraces returning to the client
// this allows for a graceful server error to be returned
LOGGER.warn("RuntimeException on executing query", e);
response.getProcessingDetails().add(new ProcessingDetailsImpl(sourceId, e));
}
long estimatedTime = System.currentTimeMillis() - startTime;
response.getProperties().put("elapsed", estimatedTime);
return response;
}
private QueryResponse getEmptyResponse(String sourceId) {
// No query was specified
QueryRequest queryRequest = new QueryRequestImpl(null, false,
Collections.singletonList(sourceId), null);
// Create a dummy QueryResponse with zero results
return new QueryResponseImpl(queryRequest, new ArrayList<Result>(), 0);
}
private JSONObject transform(Search search, SearchRequest searchRequest) throws
CatalogTransformerException {
SourceResponse upstreamResponse = search.getCompositeQueryResponse();
Map<String, MetacardType> metaTypes = new HashMap<String, MetacardType>();
if (upstreamResponse == null) {
throw new CatalogTransformerException(
"Cannot transform null " + SourceResponse.class.getName());
}
JSONObject rootObject = new JSONObject();
addObject(rootObject, Search.HITS, search.getHits());
addObject(rootObject, Search.ID, searchRequest.getId());
addObject(rootObject, Search.RESULTS,
getResultList(upstreamResponse.getResults(), metaTypes));
addObject(rootObject, Search.STATUS, getQueryStatus(search.getQueryStatus()));
addObject(rootObject, Search.METACARD_TYPES, getMetacardTypes(metaTypes.values()));
LOGGER.debug(rootObject.toJSONString());
return rootObject;
}
private JSONArray getQueryStatus(Map<String, QueryStatus> queryStatus) {
JSONArray statuses = new JSONArray();
for (String key : queryStatus.keySet()) {
QueryStatus status = queryStatus.get(key);
JSONObject statusObject = new JSONObject();
addObject(statusObject, Search.ID, status.getSourceId());
if (status.isDone()) {
addObject(statusObject, Search.RESULTS, status.getResultCount());
addObject(statusObject, Search.HITS, status.getHits());
addObject(statusObject, Search.ELAPSED, status.getElapsed());
}
addObject(statusObject, Search.STATE, status.getState());
statuses.add(statusObject);
}
return statuses;
}
private JSONArray getResultList(List<Result> results,
Map<String, MetacardType> metaTypes) throws CatalogTransformerException {
JSONArray resultsList = new JSONArray();
if (results != null) {
for (Result result : results) {
if (result == null) {
throw new CatalogTransformerException(
"Cannot transform null " + Result.class.getName());
}
JSONObject jsonObj = convertToJSON(result, metaTypes);
if (jsonObj != null) {
resultsList.add(jsonObj);
}
}
}
return resultsList;
}
private JSONObject convertToJSON(Result result, Map<String, MetacardType> metaTypes) throws
CatalogTransformerException {
JSONObject rootObject = new JSONObject();
addObject(rootObject, Search.DISTANCE, result.getDistanceInMeters());
addObject(rootObject, Search.RELEVANCE, result.getRelevanceScore());
org.json.simple.JSONObject metacardJson = GeoJsonMetacardTransformer
.convertToJSON(result.getMetacard());
metacardJson.put(Search.ACTIONS, getActions(result.getMetacard()));
Attribute cachedDate = result.getMetacard().getAttribute(Search.CACHED);
if (cachedDate != null && cachedDate.getValue() != null) {
metacardJson.put(Search.CACHED,
ISO_8601_DATE_FORMAT.print(new DateTime(cachedDate.getValue())));
} else {
metacardJson.put(Search.CACHED, ISO_8601_DATE_FORMAT.print(new DateTime()));
}
addObject(rootObject, Search.METACARD, metacardJson);
if (result.getMetacard().getMetacardType() != null && !StringUtils
.isBlank(result.getMetacard().getMetacardType().getName())) {
metaTypes.put(result.getMetacard().getMetacardType().getName(),
result.getMetacard().getMetacardType());
}
return rootObject;
}
private JSONArray getActions(Metacard metacard) {
JSONArray actionsJson = new JSONArray();
List<Action> actions = actionRegistry.list(metacard);
for (Action action : actions) {
JSONObject actionJson = new JSONObject();
actionJson.put(Search.ACTIONS_ID, action.getId());
actionJson.put(Search.ACTIONS_TITLE, action.getTitle());
actionJson.put(Search.ACTIONS_DESCRIPTION, action.getDescription());
actionJson.put(Search.ACTIONS_URL, action.getUrl());
actionsJson.add(actionJson);
}
return actionsJson;
}
private JSONObject getMetacardTypes(Collection<MetacardType> types) throws
CatalogTransformerException {
JSONObject typesObject = new JSONObject();
for (MetacardType type : types) {
JSONObject typeObj = convertToJSON(type);
if (typeObj != null) {
typesObject.put(type.getName(), typeObj);
}
}
return typesObject;
}
private JSONObject convertToJSON(MetacardType metacardType) throws CatalogTransformerException {
JSONObject fields = new JSONObject();
for (AttributeDescriptor descriptor : metacardType.getAttributeDescriptors()) {
JSONObject description = new JSONObject();
description.put("format", descriptor.getType().getAttributeFormat().toString());
description.put("indexed", descriptor.isIndexed());
fields.put(descriptor.getName(), description);
}
return fields;
}
public CatalogFramework getFramework() {
return framework;
}
public synchronized void setBayeuxServer(BayeuxServer bayeuxServer) {
this.bayeuxServer = bayeuxServer;
}
public void setCacheDisabled(Boolean cacheDisabled) {
this.cacheDisabled = cacheDisabled;
}
// Override for unit testing
ExecutorService getExecutorService() {
return Executors.newCachedThreadPool();
}
}