ParallelQueryImporter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.cyclop.service.importer.intern;

import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import javax.inject.Inject;
import javax.inject.Named;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.cyclop.model.CqlQuery;
import org.cyclop.model.CqlQueryType;
import org.cyclop.model.QueryHistory;
import org.cyclop.service.cassandra.CassandraSession;
import org.cyclop.service.importer.QueryImporter;
import org.cyclop.service.importer.ResultWriter;
import org.cyclop.service.importer.model.ImportConfig;
import org.cyclop.service.queryprotocoling.HistoryService;
import org.cyclop.validation.EnableValidation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.datastax.driver.core.Session;
import com.google.common.collect.ImmutableList;

/** @author Maciej Miklas */
@Named(QueryImporter.IMPL_PARALLEL)
@EnableValidation
public class ParallelQueryImporter extends AbstractImporter {

	private final static Logger LOG = LoggerFactory.getLogger(ParallelQueryImporter.class);

	@Inject
	protected HistoryService historyService;

	@Inject
	private CassandraSession session;

	@Inject
	@Named("importExecutor")
	private ExecutorService executor;

	@Override
	void execImport(Scanner scanner, ResultWriter resultWriter, StatsCollector status, ImportConfig iconfig) {
		ImmutableList<CqlQuery> queries = parse(scanner);
		if (queries.isEmpty()) {
			LOG.debug("No data to import");
			return;
		}

		QueryHistory history = historyService.read();

		List<Future<Void>> futures = startWorkers(queries, resultWriter, status, iconfig, history);
		waitForImport(futures);

		if (iconfig.isUpdateHistory()) {
			historyService.store(history);
		}
	}

	private List<Future<Void>> startWorkers(ImmutableList<CqlQuery> queries, ResultWriter resultWriter,
			StatsCollector status, ImportConfig iconfig, QueryHistory history) {

		final int queriesSize = queries.size();
		final int proThread = Math.max(1, queriesSize / conf.queryImport.maxThreadsProImport);

		LOG.debug("Starting parallel import for {} queries, {} pro thread", queriesSize, proThread);
		Session cassSession = session.getSession();

		List<Future<Void>> futures = new ArrayList<>();
		int startIndex = 0;
		int remaining = queriesSize;
		while (remaining > 0) {
			remaining -= proThread;
			int amount = remaining > 0 ? proThread : remaining + proThread;

			LOG.debug("Starting thread with start index: {} and amount: {}, remaining: {}", startIndex, amount,
					remaining);

			ImportWorker task = new ImportWorker(startIndex, amount, queries, status, iconfig, resultWriter,
					cassSession, history);

			Future<Void> future = executor.submit(task);
			futures.add(future);
			startIndex += amount;
		}
		return futures;
	}

	private void waitForImport(List<Future<Void>> futures) {
		LOG.debug("Tasks submitted - waiting for results");
		for (Future<Void> future : futures) {
			try {
				future.get();
			} catch (InterruptedException e) {
				Thread.interrupted();
				LOG.warn("Import executor interrupted", e);
			} catch (Exception e) {
				LOG.error("Import executor error", e);
			}
		}
	}

	private ImmutableList<CqlQuery> parse(Scanner scanner) {
		StopWatch timer = null;
		if (LOG.isDebugEnabled()) {
			timer = new StopWatch();
			timer.start();
		}

		ImmutableList.Builder<CqlQuery> build = ImmutableList.builder();
		while (scanner.hasNext()) {
			String nextStr = StringUtils.trimToNull(scanner.next());
			if (nextStr == null) {
				continue;
			}
			CqlQuery query = new CqlQuery(CqlQueryType.UNKNOWN, nextStr);
			build.add(query);
		}
		ImmutableList<CqlQuery> res = build.build();

		if (LOG.isDebugEnabled()) {
			timer.stop();
			LOG.debug("Parsed {} queries in {}", res.size(), timer.toString());
		}
		return res;
	}

}