ShardIndexingService.java

/*
 * Licensed to ElasticSearch and Shay Banon under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. ElasticSearch licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.indexing;

import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;

/**
 */
public class ShardIndexingService extends AbstractIndexShardComponent {

    private final StatsHolder totalStats = new StatsHolder();

    private volatile Map<String, StatsHolder> typesStats = ImmutableMap.of();

    private CopyOnWriteArrayList<IndexingOperationListener> listeners = null;

    @Inject
    public ShardIndexingService(ShardId shardId, @IndexSettings Settings indexSettings) {
        super(shardId, indexSettings);
    }

    /**
     * Returns the stats, including type specific stats. If the types are null/0 length, then nothing
     * is returned for them. If they are set, then only types provided will be returned, or
     * <tt>_all</tt> for all types.
     */
    public IndexingStats stats(String... types) {
        IndexingStats.Stats total = totalStats.stats();
        Map<String, IndexingStats.Stats> typesSt = null;
        if (types != null && types.length > 0) {
            if (types.length == 1 && types[0].equals("_all")) {
                typesSt = new HashMap<String, IndexingStats.Stats>(typesStats.size());
                for (Map.Entry<String, StatsHolder> entry : typesStats.entrySet()) {
                    typesSt.put(entry.getKey(), entry.getValue().stats());
                }
            } else {
                typesSt = new HashMap<String, IndexingStats.Stats>(types.length);
                for (String type : types) {
                    StatsHolder statsHolder = typesStats.get(type);
                    if (statsHolder != null) {
                        typesSt.put(type, statsHolder.stats());
                    }
                }
            }
        }
        return new IndexingStats(total, typesSt);
    }

    public synchronized void addListener(IndexingOperationListener listener) {
        if (listeners == null) {
            listeners = new CopyOnWriteArrayList<IndexingOperationListener>();
        }
        listeners.add(listener);
    }

    public synchronized void removeListener(IndexingOperationListener listener) {
        if (listeners == null) {
            return;
        }
        listeners.remove(listener);
        if (listeners.isEmpty()) {
            listeners = null;
        }
    }

    public Engine.Create preCreate(Engine.Create create) {
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                create = listener.preCreate(create);
            }
        }
        return create;
    }

    public void postCreateUnderLock(Engine.Create create) {
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                try {
                    listener.postCreateUnderLock(create);
                } catch (Exception e) {
                    logger.warn("post listener [{}] failed", e, listener);
                }
            }
        }
    }

    public void postCreate(Engine.Create create) {
        long took = create.endTime() - create.startTime();
        totalStats.indexMetric.inc(took);
        typeStats(create.type()).indexMetric.inc(took);
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                try {
                    listener.postCreate(create);
                } catch (Exception e) {
                    logger.warn("post listener [{}] failed", e, listener);
                }
            }
        }
    }

    public Engine.Index preIndex(Engine.Index index) {
        totalStats.indexCurrent.inc();
        typeStats(index.type()).indexCurrent.inc();
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                index = listener.preIndex(index);
            }
        }
        return index;
    }

    public void postIndexUnderLock(Engine.Index index) {
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                try {
                    listener.postIndexUnderLock(index);
                } catch (Exception e) {
                    logger.warn("post listener [{}] failed", e, listener);
                }
            }
        }
    }

    public void postIndex(Engine.Index index) {
        long took = index.endTime() - index.startTime();
        totalStats.indexMetric.inc(took);
        totalStats.indexCurrent.dec();
        StatsHolder typeStats = typeStats(index.type());
        typeStats.indexMetric.inc(took);
        typeStats.indexCurrent.dec();
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                try {
                    listener.postIndex(index);
                } catch (Exception e) {
                    logger.warn("post listener [{}] failed", e, listener);
                }
            }
        }
    }

    public void failedIndex(Engine.Index index) {
        totalStats.indexCurrent.dec();
        typeStats(index.type()).indexCurrent.dec();
    }

    public Engine.Delete preDelete(Engine.Delete delete) {
        totalStats.deleteCurrent.inc();
        typeStats(delete.type()).deleteCurrent.inc();
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                delete = listener.preDelete(delete);
            }
        }
        return delete;
    }

    public void postDeleteUnderLock(Engine.Delete delete) {
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                try {
                    listener.postDeleteUnderLock(delete);
                } catch (Exception e) {
                    logger.warn("post listener [{}] failed", e, listener);
                }
            }
        }
    }

    public void postDelete(Engine.Delete delete) {
        long took = delete.endTime() - delete.startTime();
        totalStats.deleteMetric.inc(took);
        totalStats.deleteCurrent.dec();
        StatsHolder typeStats = typeStats(delete.type());
        typeStats.deleteMetric.inc(took);
        typeStats.deleteCurrent.dec();
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                try {
                    listener.postDelete(delete);
                } catch (Exception e) {
                    logger.warn("post listener [{}] failed", e, listener);
                }
            }
        }
    }

    public void failedDelete(Engine.Delete delete) {
        totalStats.deleteCurrent.dec();
        typeStats(delete.type()).deleteCurrent.dec();
    }

    public Engine.DeleteByQuery preDeleteByQuery(Engine.DeleteByQuery deleteByQuery) {
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                deleteByQuery = listener.preDeleteByQuery(deleteByQuery);
            }
        }
        return deleteByQuery;
    }

    public void postDeleteByQuery(Engine.DeleteByQuery deleteByQuery) {
        if (listeners != null) {
            for (IndexingOperationListener listener : listeners) {
                listener.postDeleteByQuery(deleteByQuery);
            }
        }
    }

    public void clear() {
        totalStats.clear();
        synchronized (this) {
            if (!typesStats.isEmpty()) {
                MapBuilder<String, StatsHolder> typesStatsBuilder = MapBuilder.newMapBuilder();
                for (Map.Entry<String, StatsHolder> typeStats : typesStats.entrySet()) {
                    if (typeStats.getValue().totalCurrent() > 0) {
                        typeStats.getValue().clear();
                        typesStatsBuilder.put(typeStats.getKey(), typeStats.getValue());
                    }
                }
                typesStats = typesStatsBuilder.immutableMap();
            }
        }
    }

    private StatsHolder typeStats(String type) {
        StatsHolder stats = typesStats.get(type);
        if (stats == null) {
            synchronized (this) {
                stats = typesStats.get(type);
                if (stats == null) {
                    stats = new StatsHolder();
                    typesStats = MapBuilder.newMapBuilder(typesStats).put(type, stats).immutableMap();
                }
            }
        }
        return stats;
    }

    static class StatsHolder {
        public final MeanMetric indexMetric = new MeanMetric();
        public final MeanMetric deleteMetric = new MeanMetric();
        public final CounterMetric indexCurrent = new CounterMetric();
        public final CounterMetric deleteCurrent = new CounterMetric();

        public IndexingStats.Stats stats() {
            return new IndexingStats.Stats(
                    indexMetric.count(), TimeUnit.NANOSECONDS.toMillis(indexMetric.sum()), indexCurrent.count(),
                    deleteMetric.count(), TimeUnit.NANOSECONDS.toMillis(deleteMetric.sum()), deleteCurrent.count());
        }

        public long totalCurrent() {
            return indexCurrent.count() + deleteMetric.count();
        }

        public void clear() {
            indexMetric.clear();
            deleteMetric.clear();
        }
    }
}