AbstractFederationStrategy.java

/**
 * Copyright (c) Codice Foundation
 * <p/>
 * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser
 * General Public License as published by the Free Software Foundation, either version 3 of the
 * License, or any later version.
 * <p/>
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details. A copy of the GNU Lesser General Public License
 * is distributed along with this program and can be found at
 * <http://www.gnu.org/licenses/lgpl.html>.
 */
package ddf.catalog.federation;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import ddf.catalog.data.Result;
import ddf.catalog.operation.Query;
import ddf.catalog.operation.QueryImpl;
import ddf.catalog.operation.QueryRequest;
import ddf.catalog.operation.QueryRequestImpl;
import ddf.catalog.operation.QueryResponse;
import ddf.catalog.operation.QueryResponseImpl;
import ddf.catalog.operation.SourceResponse;
import ddf.catalog.plugin.PluginExecutionException;
import ddf.catalog.plugin.PostFederatedQueryPlugin;
import ddf.catalog.plugin.PreFederatedQueryPlugin;
import ddf.catalog.plugin.StopProcessingException;
import ddf.catalog.source.Source;

/**
 * This class serves as a base implementation of the {@link FederationStrategy}
 * interface. Other classes can extend this class to create specific attributes
 * to sort by.
 *
 * @deprecated As of release 2.3.0, replaced by
 *             ddf.catalog.federation.base.AbstractFederationStrategy
 */
@Deprecated
public abstract class AbstractFederationStrategy implements FederationStrategy {
    private static final Logger LOGGER = LoggerFactory.getLogger(AbstractFederationStrategy.class);

    // private static XLogger LOGGER = new
    // XLogger(LoggerFactory.getLogger(AbstractFederationStrategy.class));
    private static final String CLASS_NAME = AbstractFederationStrategy.class.getName();

    private static final int DEFAULT_MAX_START_INDEX = 50000;

    /**
     * The {@link List} of pre-federated query plugins to execute on the query request before the
     * query is executed on the {@link Source}.
     */
    protected List<PreFederatedQueryPlugin> preQuery;

    /**
     * The {@link List} of post-federated query plugins to execute on the query request after the
     * query is executed on the {@link Source}.
     */
    protected List<PostFederatedQueryPlugin> postQuery;

    private ExecutorService queryExecutorService;

    private int maxStartIndex;

    /**
     * Instantiates an {@code AbstractFederationStrategy} with the provided {@link ExecutorService}.
     *
     * @param queryExecutorService
     *            the {@link ExecutorService} for queries
     * @deprecated - only to provide support for deprecated FifoFederationStrategy and
     *             SortedFederationStrategy
     */
    public AbstractFederationStrategy(ExecutorService queryExecutorService) {
        this(queryExecutorService, new ArrayList<PreFederatedQueryPlugin>(),
                new ArrayList<PostFederatedQueryPlugin>());
    }

    /**
     * Instantiates an {@code AbstractFederationStrategy} with the provided {@link ExecutorService}.
     *
     * @param queryExecutorService
     *            the {@link ExecutorService} for queries
     */
    public AbstractFederationStrategy(ExecutorService queryExecutorService,
            List<PreFederatedQueryPlugin> preQuery, List<PostFederatedQueryPlugin> postQuery) {
        this.queryExecutorService = queryExecutorService;
        this.preQuery = preQuery;
        this.postQuery = postQuery;
        this.maxStartIndex = DEFAULT_MAX_START_INDEX;
    }

    /**
     * Creates the monitor for federated queries.
     *
     * @param queryExecutorService
     * @param futures
     * @param returnResults
     *            the query results
     * @param query
     * @return the {@link Runnable}
     */
    protected abstract Runnable createMonitor(ExecutorService queryExecutorService,
            Map<Source, Future<SourceResponse>> futures, QueryResponseImpl returnResults,
            Query query);

    @Override
    public QueryResponse federate(List<Source> sources, final QueryRequest queryRequest) {
        final String methodName = "federate";
        LOGGER.trace("ENTERING: {}", methodName);
        if (LOGGER.isDebugEnabled()) {
            for (Source source : sources) {
                if (source != null) {
                    LOGGER.debug("source to query: {}", source.getId());
                }
            }
        }

        Query originalQuery = queryRequest.getQuery();

        int offset = originalQuery.getStartIndex();
        final int pageSize = originalQuery.getPageSize();

        // limit offset to max value
        if (offset > this.maxStartIndex) {
            offset = this.maxStartIndex;
        }

        final QueryResponseImpl queryResponseQueue = new QueryResponseImpl(queryRequest, null);

        Map<Source, Future<SourceResponse>> futures = new HashMap<Source, Future<SourceResponse>>();

        Query modifiedQuery = getModifiedQuery(originalQuery, sources.size(), offset, pageSize);
        QueryRequest modifiedQueryRequest = new QueryRequestImpl(modifiedQuery,
                queryRequest.isEnterprise(), queryRequest.getSourceIds(),
                queryRequest.getProperties());

        // Do NOT call source.isAvailable() when checking sources
        for (final Source source : sources) {
            if (source != null) {
                if (!futures.containsKey(source)) {
                    LOGGER.debug("running query on source: " + source.getId());

                    try {
                        for (PreFederatedQueryPlugin service : preQuery) {
                            try {
                                modifiedQueryRequest = service
                                        .process(source, modifiedQueryRequest);
                            } catch (PluginExecutionException e) {
                                LOGGER.warn("Error executing PreFederatedQueryPlugin: " + e
                                                .getMessage(), e);
                            }
                        }
                    } catch (StopProcessingException e) {
                        LOGGER.warn("Plugin stopped processing: ", e);
                    }

                    futures.put(source, queryExecutorService
                            .submit(new CallableSourceResponse(source,
                                    modifiedQueryRequest.getQuery(),
                                    modifiedQueryRequest.getProperties())));
                } else {
                    LOGGER.warn("Duplicate source found with name " + source.getId()
                            + ". Ignoring second one.");
                }
            }
        }

        QueryResponseImpl offsetResults = null;
        // If there are offsets and more than one source, we have to get all the
        // results back and then
        // transfer them into a different Queue. That is what the
        // OffsetResultHandler does.
        if (offset > 1 && sources.size() > 1) {
            offsetResults = new QueryResponseImpl(queryRequest, null);
            queryExecutorService
                    .submit(new OffsetResultHandler(queryResponseQueue, offsetResults, pageSize,
                            offset));
        }

        queryExecutorService.submit(createMonitor(queryExecutorService, futures, queryResponseQueue,
                modifiedQueryRequest.getQuery()));

        QueryResponse queryResponse = null;
        if (offset > 1 && sources.size() > 1) {
            queryResponse = offsetResults;
            LOGGER.debug("returning offsetResults");
        } else {
            queryResponse = queryResponseQueue;
            LOGGER.debug("returning returnResults: {}", queryResponse);
        }

        if (null != queryResponse) {
            try {
                for (PostFederatedQueryPlugin service : postQuery) {
                    try {
                        queryResponse = service.process(queryResponse);
                    } catch (PluginExecutionException e) {
                        LOGGER.warn("Error executing PostFederatedQueryPlugin: " + e.getMessage(),
                                e);
                    }
                }
            } catch (StopProcessingException e) {
                LOGGER.warn("Plugin stopped processing: ", e);
            }
        } else {
            LOGGER.warn("No QueryResponse for PostFederatedQueryPlugins to process");
        }

        LOGGER.debug("returning Query Results: {}", queryResponse);
        LOGGER.trace("EXITING: {}.federate", CLASS_NAME);

        return queryResponse;
    }

    private Query getModifiedQuery(Query originalQuery, int numberOfSources, int offset,
            int pageSize) {

        Query query = null;

        // If offset is not specified, our offset is 1
        if (offset > 1 && numberOfSources > 1) {

            final int modifiedOffset = 1;
            int modifiedPageSize = computeModifiedPageSize(offset, pageSize);

            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("Creating new query for federated sources to query each source from "
                        + modifiedOffset + " to " + modifiedPageSize + ".");
                LOGGER.debug("original offset: " + offset);
                LOGGER.debug("original page size: " + pageSize);
                LOGGER.debug("modified offset: " + modifiedOffset);
                LOGGER.debug("modified page size: " + modifiedPageSize);
            }

            /**
             * Federated sources always query from offset of 1. When all query results are received
             * from all federated sources and merged together - then the offset is applied.
             *
             */
            query = new QueryImpl(originalQuery, modifiedOffset, modifiedPageSize,
                    originalQuery.getSortBy(), originalQuery.requestsTotalResultsCount(),
                    originalQuery.getTimeoutMillis());
        } else {
            query = originalQuery;
        }

        return query;
    }

    /**
     * Base 1 offset, hence page size is one less.
     */
    private int computeModifiedPageSize(int offset, int pageSize) {
        return offset + pageSize - 1;
    }

    /**
     * To be set via Spring/Blueprint
     *
     * @param maxStartIndex
     *            the new default max start index value
     */
    public void setMaxStartIndex(int maxStartIndex) {
        LOGGER.debug("Current max start index: " + this.maxStartIndex);
        this.maxStartIndex = DEFAULT_MAX_START_INDEX;

        if (maxStartIndex > 0) {
            this.maxStartIndex = maxStartIndex;
            LOGGER.debug("New max start index: " + this.maxStartIndex);
        } else {
            LOGGER.debug(
                    "Invalid max start index input. Reset to default value: " + this.maxStartIndex);
        }
    }

    private static class CallableSourceResponse implements Callable<SourceResponse> {

        private Query query = null;

        private Source source = null;

        private Map<String, Serializable> properties = null;

        public CallableSourceResponse(Source source, Query query,
                Map<String, Serializable> properties) {
            this.source = source;
            this.query = query;
            this.properties = properties;
        }

        @Override
        public SourceResponse call() throws Exception {
            return source.query(new QueryRequestImpl(query, properties));
        }

    }

    private static class OffsetResultHandler implements Runnable {

        private QueryResponseImpl originalResults = null;

        private QueryResponseImpl offsetResultQueue = null;

        private int pageSize = 0;

        private int offset = 1;

        private OffsetResultHandler(QueryResponseImpl originalResults,
                QueryResponseImpl offsetResultQueue, int pageSize, int offset) {
            this.originalResults = originalResults;
            this.offsetResultQueue = offsetResultQueue;
            this.pageSize = pageSize;
            this.offset = offset;

        }

        @Override
        public void run() {
            int queryResultIndex = 1;
            int resultsSent = 0;
            Result result;

            while (resultsSent < pageSize && originalResults.hasMoreResults()
                    && (result = originalResults.take()) != null) {
                if (queryResultIndex >= offset) {
                    offsetResultQueue.addResult(result, false);
                    resultsSent++;
                }
                queryResultIndex++;
            }

            LOGGER.debug("Closing Queue and setting the total count");
            offsetResultQueue.setHits(originalResults.getHits());
            offsetResultQueue.closeResultQueue();
        }
    }

}