SourceMetricsImpl.java

/**
 * Copyright (c) Codice Foundation
 * <p>
 * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser
 * General Public License as published by the Free Software Foundation, either version 3 of the
 * License, or any later version.
 * <p>
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details. A copy of the GNU Lesser General Public License
 * is distributed along with this program and can be found at
 * <http://www.gnu.org/licenses/lgpl.html>.
 */
package ddf.catalog.metrics.source;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.codahale.metrics.Histogram;
import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;

import ddf.catalog.data.Result;
import ddf.catalog.operation.ProcessingDetails;
import ddf.catalog.operation.QueryRequest;
import ddf.catalog.operation.QueryResponse;
import ddf.catalog.plugin.PluginExecutionException;
import ddf.catalog.plugin.PostFederatedQueryPlugin;
import ddf.catalog.plugin.PreFederatedQueryPlugin;
import ddf.catalog.plugin.StopProcessingException;
import ddf.catalog.source.CatalogProvider;
import ddf.catalog.source.FederatedSource;
import ddf.catalog.source.Source;
import ddf.metrics.collector.rrd4j.RrdJmxCollector;

/**
 * This class manages the metrics for individual {@link CatalogProvider} and {@link FederatedSource}
 * {@link Source}s. These metrics currently include the count of queries, results per query, and
 * exceptions per {@link Source}.
 *
 * The metrics and their associated {@link ddf.metrics.collector.JmxCollector}s are created when the {@link Source} is
 * created and deleted when the {@link Source} is deleted. (The associated RRD file remains
 * available indefinitely and accessible from the Metrics tab in the Web Admin console unless an
 * administrator manually deletes it).
 *
 * If a {@link Source} is renamed, i.e., its ID changed, then the {@link Source}'s existing metrics'
 * MBeans and {@link ddf.metrics.collector.JmxCollector}s are deleted and new metrics created using the new {@link Source}
 * 's ID. However, the RRD file for the {@link Source}'s previous source ID remains available and
 * accessible from the Metrics tab in the Web Admin console unless an administrator manually deletes
 * it.
 *
 * @author rodgersh
 * @author ddf.isgs@lmco.com
 *
 */
public class SourceMetricsImpl implements PreFederatedQueryPlugin, PostFederatedQueryPlugin {

    /**
     * Package name for the JMX MBean where metrics for {@link Source}s are stored.
     */
    public static final String MBEAN_PACKAGE_NAME = "ddf.metrics.catalog.source";

    /**
     * Name of the JMX MBean scope for source-level metrics tracking exceptions while querying a
     * specific {@link Source}
     */
    public static final String EXCEPTIONS_SCOPE = "Exceptions";

    /**
     * Name of the JMX MBean scope for source-level metrics tracking query count while querying a
     * specific {@link Source}
     */
    public static final String QUERIES_SCOPE = "Queries";

    /**
     * Name of the JMX MBean scope for source-level metrics tracking total results returned while
     * querying a specific {@link Source}
     */
    public static final String QUERIES_TOTAL_RESULTS_SCOPE = "Queries.TotalResults";

    public static final String DERIVE_DATA_SOURCE_TYPE = "DERIVE";

    public static final String GAUGE_DATA_SOURCE_TYPE = "GAUGE";

    public static final String COUNT_MBEAN_ATTRIBUTE_NAME = "Count";

    public static final String MEAN_MBEAN_ATTRIBUTE_NAME = "Mean";

    private static final Logger LOGGER = LoggerFactory.getLogger(SourceMetricsImpl.class);

    private static final String ALPHA_NUMERIC_REGEX = "[^a-zA-Z0-9]";

    private final MetricRegistry metricsRegistry = new MetricRegistry();

    private final JmxReporter reporter = JmxReporter.forRegistry(metricsRegistry)
            .inDomain(MBEAN_PACKAGE_NAME)
            .build();

    // Map of sourceId to Source's metric data
    protected Map<String, SourceMetric> metrics = new HashMap<String, SourceMetric>();

    // Injected list of CatalogProviders and FederatedSources
    // that is kept updated by container, e.g., with latest sourceIds
    private List<CatalogProvider> catalogProviders = new ArrayList<CatalogProvider>();

    private List<FederatedSource> federatedSources = new ArrayList<FederatedSource>();

    // Map of Source to sourceId - used to detect if sourceId has been changed since last metric
    // update
    private Map<Source, String> sourceToSourceIdMap = new HashMap<Source, String>();

    private ExecutorService executorPool;

    public List<CatalogProvider> getCatalogProviders() {
        return catalogProviders;
    }

    public void setCatalogProviders(List<CatalogProvider> catalogProviders) {
        this.catalogProviders = catalogProviders;
    }

    public List<FederatedSource> getFederatedSources() {
        return federatedSources;
    }

    public void setFederatedSources(List<FederatedSource> federatedSources) {
        this.federatedSources = federatedSources;
    }

    public void init() {
        LOGGER.trace("INSIDE: init");

        reporter.start();
    }

    public void destroy() {
        LOGGER.trace("INSIDE: destroy");

        reporter.stop();
    }

    // PreFederatedQuery
    @Override
    public QueryRequest process(Source source, QueryRequest input)
            throws PluginExecutionException, StopProcessingException {

        LOGGER.trace("ENTERING: process (for PreFederatedQueryPlugin)");

        // Number of Queries metric per Source
        updateMetric(source.getId(), QUERIES_SCOPE, 1);

        LOGGER.trace("EXITING: process (for PreFederatedQueryPlugin)");

        return input;
    }

    // PostFederatedQuery
    @Override
    public QueryResponse process(QueryResponse input)
            throws PluginExecutionException, StopProcessingException {

        LOGGER.trace("ENTERING: process (for PostFederatedQueryPlugin)");

        if (null != input) {
            Set<ProcessingDetails> processingDetails = input.getProcessingDetails();
            List<Result> results = input.getResults();

            // Total Exceptions metric per Source
            Iterator<ProcessingDetails> iterator = processingDetails.iterator();
            while (iterator.hasNext()) {
                ProcessingDetails next = iterator.next();
                if (next != null && next.getException() != null) {
                    String sourceId = next.getSourceId();
                    updateMetric(sourceId, EXCEPTIONS_SCOPE, 1);
                }
            }

            Map<String, Integer> totalHitsPerSource = new HashMap<String, Integer>();

            for (Result result : results) {
                String sourceId = result.getMetacard()
                        .getSourceId();
                if (totalHitsPerSource.containsKey(sourceId)) {
                    totalHitsPerSource.put(sourceId, totalHitsPerSource.get(sourceId) + 1);
                } else {
                    // First detection of this new source ID in the results list -
                    // initialize the Total Query Result Count for this Source
                    totalHitsPerSource.put(sourceId, 1);
                }
            }

            // Total Query Results metric per Source
            for (Map.Entry<String, Integer> source : totalHitsPerSource.entrySet()) {
                updateMetric(source.getKey(), QUERIES_TOTAL_RESULTS_SCOPE, source.getValue());
            }
        }

        LOGGER.trace("EXITING: process (for PostFederatedQueryPlugin)");

        return input;
    }

    public void updateMetric(String sourceId, String name, int incrementAmount) {

        LOGGER.debug("sourceId = {},   name = {}", sourceId, name);

        if (StringUtils.isBlank(sourceId) || StringUtils.isBlank(name)) {
            return;
        }

        String mapKey = sourceId + "." + name;
        SourceMetric sourceMetric = metrics.get(mapKey);

        if (sourceMetric == null) {
            LOGGER.debug("sourceMetric is null for {} - creating metric now", mapKey);
            // Loop through list of all sources until find the sourceId whose metric is being
            // updated
            boolean created = createMetric(catalogProviders, sourceId);
            if (!created) {
                createMetric(federatedSources, sourceId);
            }
            sourceMetric = metrics.get(mapKey);
        }

        // If this metric already exists, then just update its MBean
        if (sourceMetric != null) {
            LOGGER.debug("CASE 1: Metric already exists for {}", mapKey);
            if (sourceMetric.isHistogram()) {
                Histogram metric = (Histogram) sourceMetric.getMetric();
                LOGGER.debug(
                        "Updating histogram metric {} by amount of {}", name, incrementAmount);
                metric.update(incrementAmount);
            } else {
                Meter metric = (Meter) sourceMetric.getMetric();
                LOGGER.debug("Updating metric {} by amount of {}", name, incrementAmount);
                metric.mark(incrementAmount);
            }
            return;
        }
    }

    private boolean createMetric(List<? extends Source> sources, String sourceId) {
        for (Source source : sources) {
            if (source.getId()
                    .equals(sourceId)) {
                LOGGER.debug("Found sourceId = {} in sources list", sourceId);
                if (sourceToSourceIdMap.containsKey(source)) {
                    // Source's ID must have changed since it is in this map but not in the metrics
                    // map
                    // Delete SourceMetrics for Source's "old" sourceId
                    String oldSourceId = sourceToSourceIdMap.get(source);
                    LOGGER.debug("CASE 2: source {} exists but has oldSourceId = {}", sourceId, oldSourceId);
                    deleteMetric(oldSourceId, QUERIES_TOTAL_RESULTS_SCOPE);
                    deleteMetric(oldSourceId, QUERIES_SCOPE);
                    deleteMetric(oldSourceId, EXCEPTIONS_SCOPE);

                    // Create metrics for Source with new sourceId
                    createMetric(sourceId, QUERIES_TOTAL_RESULTS_SCOPE, MetricType.HISTOGRAM);
                    createMetric(sourceId, QUERIES_SCOPE, MetricType.METER);
                    createMetric(sourceId, EXCEPTIONS_SCOPE, MetricType.METER);

                    // Add Source to map with its new sourceId
                    sourceToSourceIdMap.put(source, sourceId);
                } else {
                    // This is a brand new Source - create metrics for it
                    // (Should rarely happen since Sources typically have their metrics created
                    // when the Source itself is created via the addingSource() method. This could
                    // happen if sourceId = null when Source originally created and then its metric
                    // needs updating because client, e.g., SortedFederationStrategy, knows the
                    // Source exists.)
                    LOGGER.debug("CASE 3: New source {} detected - creating metrics", sourceId);
                    createMetric(sourceId, QUERIES_TOTAL_RESULTS_SCOPE, MetricType.HISTOGRAM);
                    createMetric(sourceId, QUERIES_SCOPE, MetricType.METER);
                    createMetric(sourceId, EXCEPTIONS_SCOPE, MetricType.METER);

                    sourceToSourceIdMap.put(source, sourceId);
                }
                return true;
            }
        }

        LOGGER.debug("Did not find source {} in Sources - cannot create metrics", sourceId);

        return false;
    }

    /**
     * Creates metrics for new CatalogProvider or FederatedSource when they are initially created.
     * Metrics creation includes the JMX MBeans and associated ddf.metrics.collector.JmxCollector.
     *
     * @param source
     * @param props
     */
    public void addingSource(final Source source, Map props) {
        LOGGER.trace("ENTERING: addingSource");

        if (executorPool == null) {
            executorPool = Executors.newCachedThreadPool();
        }

        // Creating JmxCollectors for all of the source metrics can be time consuming,
        // so do this in a separate thread to prevent blacklisting by EventAdmin
        final Runnable metricsCreator = new Runnable() {
            public void run() {
                createSourceMetrics(source);
            }
        };

        LOGGER.debug("Start metricsCreator thread for Source {}", source.getId());
        executorPool.execute(metricsCreator);

        LOGGER.trace("EXITING: addingSource");
    }

    /**
     * Deletes metrics for existing CatalogProvider or FederatedSource when they are deleted.
     * Metrics deletion includes the JMX MBeans and associated ddf.metrics.collector.JmxCollector.
     *
     * @param source
     * @param props
     */
    public void deletingSource(final Source source, final Map props) {
        LOGGER.trace("ENTERING: deletingSource");

        if (source == null || StringUtils.isBlank(source.getId())) {
            LOGGER.debug("Not deleting metrics for NULL or blank source");
            return;
        }

        String sourceId = source.getId();

        LOGGER.debug("sourceId = {},    props = {}", sourceId, props);

        deleteMetric(sourceId, QUERIES_TOTAL_RESULTS_SCOPE);
        deleteMetric(sourceId, QUERIES_SCOPE);
        deleteMetric(sourceId, EXCEPTIONS_SCOPE);

        // Delete source from internal map used when updating metrics by sourceId
        sourceToSourceIdMap.remove(source);

        LOGGER.trace("EXITING: deletingSource");
    }

    // Separate, package-scope method to allow unit testing
    void createSourceMetrics(final Source source) {

        if (source == null || StringUtils.isBlank(source.getId())) {
            LOGGER.debug("Not adding metrics for NULL or blank source");
            return;
        }

        String sourceId = source.getId();

        LOGGER.debug("sourceId = {}", sourceId);

        createMetric(sourceId, QUERIES_TOTAL_RESULTS_SCOPE, MetricType.HISTOGRAM);
        createMetric(sourceId, QUERIES_SCOPE, MetricType.METER);
        createMetric(sourceId, EXCEPTIONS_SCOPE, MetricType.METER);

        // Add new source to internal map used when updating metrics by sourceId
        sourceToSourceIdMap.put(source, sourceId);
    }

    private void createMetric(String sourceId, String mbeanName, MetricType type) {

        // Create source-specific metrics for this source
        // (Must be done prior to creating metrics collector so that
        // JMX MBean exists for collector to detect).
        String key = sourceId + "." + mbeanName;

        // Do not create metric and collector if they already exist for this source.
        // (This can happen for ConnectedSources because they have the same sourceId
        // as the local catalog provider).
        if (!metrics.containsKey(key)) {
            if (type == MetricType.HISTOGRAM) {
                Histogram histogram = metricsRegistry.histogram(MetricRegistry.name(sourceId,
                        mbeanName));
                RrdJmxCollector collector = createGaugeMetricsCollector(sourceId, mbeanName);
                metrics.put(key, new SourceMetric(histogram, collector, true));
            } else if (type == MetricType.METER) {
                Meter meter = metricsRegistry.meter(MetricRegistry.name(sourceId, mbeanName));
                RrdJmxCollector collector = createCounterMetricsCollector(sourceId, mbeanName);
                metrics.put(key, new SourceMetric(meter, collector));
            } else {
                LOGGER.debug("Metric {} not created because unknown metric type {} specified.", key, type);
            }
        } else {
            LOGGER.debug("Metric {} already exists - not creating again", key);
        }
    }

    /**
     * Creates the Counter JMX Collector for an associated metric's JMX MBean.
     *
     * @param sourceId
     * @param collectorName
     * @return the ddf.metrics.collector.JmxCollector created
     */
    private RrdJmxCollector createCounterMetricsCollector(String sourceId, String collectorName) {
        return createMetricsCollector(sourceId,
                collectorName,
                COUNT_MBEAN_ATTRIBUTE_NAME,
                DERIVE_DATA_SOURCE_TYPE);
    }

    /**
     * Creates the Gauge JMX Collector for an associated metric's JMX MBean.
     *
     * @param sourceId
     * @param collectorName
     * @return the ddf.metrics.collector.JmxCollector created
     */
    private RrdJmxCollector createGaugeMetricsCollector(String sourceId, String collectorName) {
        return createMetricsCollector(sourceId,
                collectorName,
                MEAN_MBEAN_ATTRIBUTE_NAME,
                GAUGE_DATA_SOURCE_TYPE);
    }

    /**
     * Creates the JMX Collector for an associated metric's JMX MBean.
     *
     * @param sourceId
     * @param collectorName
     * @param mbeanAttributeName
     *            usually "Count" or "Mean"
     * @param dataSourceType
     *            only "DERIVE", "COUNTER" or "GAUGE" are supported
     * @return the ddf.metrics.collector.JmxCollector created
     */
    private RrdJmxCollector createMetricsCollector(String sourceId, String collectorName,
            String mbeanAttributeName, String dataSourceType) {

        LOGGER.trace(
                "ENTERING: createMetricsCollector - sourceId = {},   collectorName = {},   mbeanAttributeName = {},   dataSourceType = {}",
                sourceId,
                collectorName,
                mbeanAttributeName,
                dataSourceType);

        String rrdPath = getRrdFilename(sourceId, collectorName);

        RrdJmxCollector collector = new RrdJmxCollector(
                MBEAN_PACKAGE_NAME + ":name=" + sourceId + "." + collectorName,
                mbeanAttributeName,
                rrdPath,
                dataSourceType);
        collector.init();

        LOGGER.trace("EXITING: createMetricsCollector - sourceId = {}", sourceId);

        return collector;
    }

    protected String getRrdFilename(String sourceId, String collectorName) {

        // Based on the sourceId and collectorName, generate the name of the RRD file.
        // This RRD file will be of the form "source<sourceId><collectorName>.rrd" with
        // the non-alphanumeric characters stripped out and the next character after any
        // non-alphanumeric capitalized.
        // Example:
        // Given sourceId = dib30rhel-58 and collectorName = Queries.TotalResults
        // The resulting RRD filename would be: sourceDib30rhel58QueriesTotalResults
        String[] sourceIdParts = sourceId.split(ALPHA_NUMERIC_REGEX);
        StringBuilder newSourceIdBuilder = new StringBuilder("");
        for (String part : sourceIdParts) {
            newSourceIdBuilder.append(StringUtils.capitalize(part));
        }
        String rrdPath = "source" + newSourceIdBuilder.toString() + collectorName;
        LOGGER.debug("BEFORE: rrdPath = {}", rrdPath);

        // Sterilize RRD path name by removing any non-alphanumeric characters - this would confuse
        // the
        // URL being generated for this RRD path in the Metrics tab of Admin console.
        rrdPath = rrdPath.replaceAll(ALPHA_NUMERIC_REGEX, "");
        LOGGER.debug("AFTER: rrdPath = {}", rrdPath);

        return rrdPath;
    }

    /**
     * Delete the metric's MBean for the specified Source.
     *
     * @param sourceId
     * @param mbeanName
     */
    private void deleteMetric(String sourceId, String mbeanName) {

        String key = sourceId + "." + mbeanName;
        if (metrics.containsKey(key)) {
            metricsRegistry.remove(MetricRegistry.name(sourceId, mbeanName));
            deleteCollector(sourceId, mbeanName);
            metrics.remove(key);
        } else {
            LOGGER.debug("Did not remove metric {} because it was not in metrics map", key);
        }
    }

    /**
     * Delete the ddf.metrics.collector.JmxCollector for the specified Source and MBean.
     *
     * @param sourceId
     * @param metricName
     */
    private void deleteCollector(String sourceId, String metricName) {
        String mapKey = sourceId + "." + metricName;
        SourceMetric sourceMetric = metrics.get(mapKey);
        LOGGER.debug("Deleting {} ddf.metrics.collector.JmxCollector for source {}", metricName, sourceId);
        sourceMetric.getCollector()
                .destroy();
        metrics.remove(mapKey);
    }

    // The types of Yammer Metrics supported
    private enum MetricType {
        HISTOGRAM, METER
    }

    /**
     * Inner class POJO to maintain details of each metric for each Source.
     *
     * @author rodgersh
     *
     */
    public static class SourceMetric {

        // The Yammer Metric
        private Metric metric;

        // The ddf.metrics.collector.JmxCollector polling this metric's MBean
        private RrdJmxCollector collector;

        // Whether this metric is a Histogram or Meter
        private boolean isHistogram = false;

        public SourceMetric(Metric metric, RrdJmxCollector collector) {
            this(metric, collector, false);
        }

        public SourceMetric(Metric metric, RrdJmxCollector collector, boolean isHistogram) {
            this.metric = metric;
            this.collector = collector;
            this.isHistogram = isHistogram;
        }

        public Metric getMetric() {
            return metric;
        }

        public RrdJmxCollector getCollector() {
            return collector;
        }

        public boolean isHistogram() {
            return isHistogram;
        }

    }

}