/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.merge.scheduler;
import org.apache.lucene.index.*;
import org.apache.lucene.store.AlreadyClosedException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.merge.policy.EnableMergePolicy;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
/**
*
*/
public class ConcurrentMergeSchedulerProvider extends AbstractIndexShardComponent implements MergeSchedulerProvider {
private final int maxThreadCount;
private final int maxMergeCount;
private Set<CustomConcurrentMergeScheduler> schedulers = new CopyOnWriteArraySet<CustomConcurrentMergeScheduler>();
@Inject
public ConcurrentMergeSchedulerProvider(ShardId shardId, @IndexSettings Settings indexSettings) {
super(shardId, indexSettings);
// TODO LUCENE MONITOR this will change in Lucene 4.0
this.maxThreadCount = componentSettings.getAsInt("max_thread_count", Math.max(1, Math.min(3, Runtime.getRuntime().availableProcessors() / 2)));
this.maxMergeCount = componentSettings.getAsInt("max_merge_count", maxThreadCount + 2);
logger.debug("using [concurrent] merge scheduler with max_thread_count[{}]", maxThreadCount);
}
@Override
public MergeScheduler newMergeScheduler() {
CustomConcurrentMergeScheduler concurrentMergeScheduler = new CustomConcurrentMergeScheduler(logger, shardId, this);
concurrentMergeScheduler.setMaxMergeCount(maxMergeCount);
concurrentMergeScheduler.setMaxThreadCount(maxThreadCount);
schedulers.add(concurrentMergeScheduler);
return concurrentMergeScheduler;
}
@Override
public MergeStats stats() {
MergeStats mergeStats = new MergeStats();
for (CustomConcurrentMergeScheduler scheduler : schedulers) {
mergeStats.add(scheduler.totalMerges(), scheduler.totalMergeTime(), scheduler.totalMergeNumDocs(), scheduler.totalMergeSizeInBytes(),
scheduler.currentMerges(), scheduler.currentMergesNumDocs(), scheduler.currentMergesSizeInBytes());
}
return mergeStats;
}
public static class CustomConcurrentMergeScheduler extends TrackingConcurrentMergeScheduler {
private final ShardId shardId;
private final ConcurrentMergeSchedulerProvider provider;
private CustomConcurrentMergeScheduler(ESLogger logger, ShardId shardId, ConcurrentMergeSchedulerProvider provider) {
super(logger);
this.shardId = shardId;
this.provider = provider;
}
@Override
public void merge(IndexWriter writer) throws CorruptIndexException, IOException {
try {
// if merge is not enabled, don't do any merging...
if (writer.getConfig().getMergePolicy() instanceof EnableMergePolicy) {
if (!((EnableMergePolicy) writer.getConfig().getMergePolicy()).isMergeEnabled()) {
return;
}
}
} catch (AlreadyClosedException e) {
// called writer#getMergePolicy can cause an AlreadyClosed failure, so ignore it
// since we are doing it on close, return here and don't do the actual merge
// since we do it outside of a lock in the RobinEngine
return;
}
try {
super.merge(writer);
} catch (IOException e) {
logger.warn("failed to merge", e);
throw e;
}
}
@Override
protected MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
MergeThread thread = super.getMergeThread(writer, merge);
thread.setName(EsExecutors.threadName(provider.indexSettings(), "[" + shardId.index().name() + "][" + shardId.id() + "]: " + thread.getName()));
return thread;
}
@Override
protected void handleMergeException(Throwable exc) {
logger.warn("failed to merge", exc);
super.handleMergeException(exc);
}
@Override
public void close() {
super.close();
provider.schedulers.remove(this);
}
}
}