ParallelJobExecutor.java

/**
 * Copyright 2011-2017 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.yaess.paralleljob;

import java.io.IOException;
import java.text.MessageFormat;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.NavigableMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.asakusafw.yaess.basic.JobExecutor;
import com.asakusafw.yaess.core.ExecutionContext;
import com.asakusafw.yaess.core.ExecutionMonitor;
import com.asakusafw.yaess.core.ExecutionScriptHandler;
import com.asakusafw.yaess.core.Job;
import com.asakusafw.yaess.core.VariableResolver;
import com.asakusafw.yaess.core.YaessLogger;
import com.asakusafw.yaess.core.util.PropertiesUtil;

/**
 * An implementation of {@link ParallelJobExecutor} which use multiple threads for each resource.
 * @since 0.2.3
 */
public class ParallelJobExecutor implements JobExecutor {

    static final YaessLogger YSLOG = new YaessParallelJobLogger(ParallelJobExecutor.class);

    static final Logger LOG = LoggerFactory.getLogger(ParallelJobExecutor.class);

    /**
     * The key prefix of multiplexity configuration each resources.
     */
    public static final String KEY_PARALLEL_PREFIX = "parallel.";

    /**
     * The default resource name.
     */
    public static final String DEFAULT_RESOURCE_ID = ExecutionScriptHandler.DEFAULT_RESOURCE_ID;

    private final ExecutorService defaultExecutor;

    private final Map<String, ExecutorService> resourceExecutors;

    /**
     * Creates a new instance.
     * @param defaultResuorce default resource multiplexity
     * @param threadConfig each resource multiplexity
     * @throws IllegalArgumentException if some parameters were {@code null}
     */
    public ParallelJobExecutor(int defaultResuorce, Map<String, Integer> threadConfig) {
        if (defaultResuorce <= 0) {
            throw new IllegalArgumentException("defaultResurce must be > 0"); //$NON-NLS-1$
        }
        if (threadConfig == null) {
            throw new IllegalArgumentException("threadConfig must not be null"); //$NON-NLS-1$
        }
        this.defaultExecutor = Executors.newFixedThreadPool(defaultResuorce, new ThreadFactory() {
            private final AtomicInteger count = new AtomicInteger();
            @Override
            public Thread newThread(Runnable r) {
                Thread thread = new Thread(r);
                thread.setName(MessageFormat.format(
                        "ParallelJobScheduler-{0}",
                        String.valueOf(count.incrementAndGet())));
                return thread;
            }
        });
        HashMap<String, ExecutorService> map = new HashMap<>();
        for (Map.Entry<String, Integer> entry : threadConfig.entrySet()) {
            String name = entry.getKey();
            Integer value = entry.getValue();
            if (value == null || value < 0) {
                throw new IllegalArgumentException(MessageFormat.format(
                        "thread config must be > 0: key={0}, value={1}",
                        name,
                        value));
            }
            map.put(name, Executors.newFixedThreadPool(value, new ThreadFactory() {
                private final AtomicInteger count = new AtomicInteger();
                @Override
                public Thread newThread(Runnable r) {
                    Thread thread = new Thread(r);
                    thread.setName(MessageFormat.format(
                            "ParallelJobScheduler-{0}-{1}",
                            name,
                            String.valueOf(count.incrementAndGet())));
                    return thread;
                }
            }));
        }
        map.put(DEFAULT_RESOURCE_ID, defaultExecutor);
        resourceExecutors = Collections.unmodifiableMap(map);
    }

    /**
     * Extracts multiplexity profiles from configuration and returns a related executor.
     * <p>
     * This operation extracts following entries from {@code configuration}:
     * </p>
     * <ul>
     * <li> {@link #KEY_PARALLEL_PREFIX parallel.<resource-name>} - </li>
     * </ul>
     * <p>
     * Profiles must be contain {@link #DEFAULT_RESOURCE_ID parallel.default}.
     * </p>
     * @param servicePrefix prefix of configuration keys
     * @param configuration target configuration
     * @param variables variable resolver
     * @return the created executor
     * @throws IllegalArgumentException if configuration is invalid
     */
    public static ParallelJobExecutor extract(
            String servicePrefix,
            Map<String, String> configuration,
            VariableResolver variables) {
        if (servicePrefix == null) {
            throw new IllegalArgumentException("servicePrefix must not be null"); //$NON-NLS-1$
        }
        if (configuration == null) {
            throw new IllegalArgumentException("configuration must not be null"); //$NON-NLS-1$
        }
        if (variables == null) {
            throw new IllegalArgumentException("variables must not be null"); //$NON-NLS-1$
        }
        NavigableMap<String, String> segment = PropertiesUtil.createPrefixMap(configuration, KEY_PARALLEL_PREFIX);
        Map<String, Integer> conf = new HashMap<>();
        for (Map.Entry<String, String> entry : segment.entrySet()) {
            String name = entry.getKey();
            String valueString = entry.getValue();
            try {
                valueString = variables.replace(valueString, true);
            } catch (IllegalArgumentException e) {
                throw new IllegalArgumentException(MessageFormat.format(
                        "Failed to resolve the profile \"{0}\": {1}",
                        servicePrefix + '.' + name,
                        valueString), e);
            }
            Integer value;
            try {
                value = Integer.valueOf(valueString);
            } catch (NumberFormatException e) {
                value = null;
            }
            if (value == null || value <= 0) {
                throw new IllegalArgumentException(MessageFormat.format(
                        "Parallel configuration \"{0}\" must be > 0: {1}",
                        servicePrefix + '.' + KEY_PARALLEL_PREFIX + name,
                        valueString));
            }
            conf.put(name, value);
        }
        LOG.debug("ParallelJobExecutor: {}", conf);
        Integer defaultValue = conf.remove(DEFAULT_RESOURCE_ID);
        if (defaultValue == null) {
            throw new IllegalArgumentException(MessageFormat.format(
                    "Default parallel configuration \"{0}\" is not defined",
                    servicePrefix + '.' + KEY_PARALLEL_PREFIX + DEFAULT_RESOURCE_ID));
        }
        return new ParallelJobExecutor(defaultValue, conf);
    }

    @Override
    public Executing submit(
            ExecutionMonitor monitor,
            ExecutionContext context,
            Job job,
            BlockingQueue<Executing> doneQueue) throws InterruptedException, IOException {
        if (monitor == null) {
            throw new IllegalArgumentException("monitor must not be null"); //$NON-NLS-1$
        }
        if (context == null) {
            throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
        }
        if (job == null) {
            throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$
        }
        String resourceId = job.getResourceId(context);
        ExecutorService executor = resourceExecutors.get(resourceId);
        if (executor == null) {
            YSLOG.warn("W01001",
                    context.getBatchId(),
                    context.getFlowId(),
                    context.getExecutionId(),
                    context.getPhase(),
                    job.getJobLabel(),
                    job.getServiceLabel(),
                    resourceId);
            LOG.debug("Resource {} is not defined: {}", resourceId, job.getId());
            executor = defaultExecutor;
        } else {
            YSLOG.info("I01001",
                    context.getBatchId(),
                    context.getFlowId(),
                    context.getExecutionId(),
                    context.getPhase(),
                    job.getJobLabel(),
                    job.getServiceLabel(),
                    resourceId);
        }
        Executing executing = new Executing(monitor, context, job, doneQueue);
        executor.execute(executing);
        return executing;
    }
}