ExecutionManager.java

/*
 * Copyright (C) 2014 Indeed Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the
 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing permissions and
 * limitations under the License.
 */
 package com.indeed.imhotep.web;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.annotation.Nonnull;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

/**
 * Keeps track of the currently running queries.
 * All public methods on this class operating on internal data structures should be marked synchronized.
 * @author vladimir
 */
@Component
public class ExecutionManager {
    private static final Logger log = Logger.getLogger(ExecutionManager.class);
    // values are used for locking to avoid concurrent processing of identical requests
    private final Map<String, CountDownLatch> queryToLock = Maps.newHashMap();
    // used to limit number of concurrent queries per user
    private final Map<String, Semaphore> userToLock = Maps.newHashMap();
    private final Set<QueryTracker> runningQueries = Sets.newHashSet();

    @Value("${user.concurrent.query.limit}")
    private int maxQueriesPerUser;

    public ExecutionManager() {
    }

    @Nonnull
    public synchronized List<QueryTracker> getRunningQueries() {
        return Lists.newArrayList(runningQueries);
    }

    /**
     * Keeps track of the query that is going to be executed and makes sure that user allocated limit is not exceeded.
     * When the query execution is completed (data is in HDFS cache) or fails, the returned Query object must be closed.
     */
    public synchronized QueryTracker queryStarted(String query, String username) throws TimeoutException {
        final CountDownLatch releaseLock;
        final CountDownLatch waitLockForQuery;
        final Semaphore waitLockForUser;
        if(queryToLock.containsKey(query)) {   // this is a duplicate query and execution will have to wait
            waitLockForQuery = queryToLock.get(query);
            waitLockForUser = null;
            releaseLock = null;
        } else {    // this is a non-duplicate query and the lock has to be released after execution is finished
            waitLockForQuery = null;
            waitLockForUser = getUserSemaphore(username);
            releaseLock = new CountDownLatch(1);
            queryToLock.put(query, releaseLock);
        }
        final QueryTracker newQueryTracker = new QueryTracker(username, query, waitLockForQuery, waitLockForUser, releaseLock, this);
        runningQueries.add(newQueryTracker);
        return newQueryTracker;
    }

    private synchronized Semaphore getUserSemaphore(String username) {
        Semaphore semaphore = userToLock.get(username);
        if(semaphore == null) {
            semaphore = new Semaphore(maxQueriesPerUser, true);
            userToLock.put(username, semaphore);
        }
        return semaphore;
    }

    private synchronized void release(QueryTracker q) {
        if(q.released) {
            return; // release called twice
        }
        q.released = true;

        if(q.releaseLock != null) { // this was the original query of this type and it's done now
            q.releaseLock.countDown();
            queryToLock.remove(q.query);
        }
        if(q.userSlotUsed && q.waitLockForUser != null) {
            q.waitLockForUser.release();
        }

        runningQueries.remove(q);
    }

    /**
     * Keeps track of the query execution.
     * Must be closed when all operations relating to the query processing are complete (including HDFS cache upload).
     */
    public class QueryTracker implements Closeable {
        private final String username;  // user running the query
        private final String query; // query text
        private final CountDownLatch waitLockForQuery;  // lock to wait on before the query starts
        private final Semaphore waitLockForUser;  // lock to wait on before the query starts
        private final CountDownLatch releaseLock;   // lock to be released when the query is done
        private final ExecutionManager owner;
        private final DateTime startedTime = DateTime.now();
        private boolean asynchronousRelease = false;
        private boolean released = false;
        private boolean userSlotUsed = false;

        private QueryTracker(String username, String query, CountDownLatch waitLockForQuery, Semaphore waitLockForUser, CountDownLatch releaseLock, ExecutionManager owner) {
            this.username = username;
            this.query = query;
            this.waitLockForQuery = waitLockForQuery;
            this.waitLockForUser = waitLockForUser;
            this.releaseLock = releaseLock;
            this.owner = owner;
        }

        public String getUsername() {
            return username;
        }

        public String getQuery() {
            return query;
        }

        public String getStartedTime() {
            return startedTime.toString();
        }

        public void acquireLocks() throws TimeoutException {
            waitForQueryLock();
            waitForUserLock();
        }

        private void waitForQueryLock() throws TimeoutException {
            if(waitLockForQuery == null) {
                return;
            }
            // same query is already being handled, waiting
            try {
                if(!waitLockForQuery.await(5, TimeUnit.MINUTES)) {
                    log.error("Reached timeout waiting for completion of: " + query);
                    throw new TimeoutException("Reached timeout (5 min) waiting for completion of original execution of the query");
                }
            } catch (InterruptedException ignored) {
                throw new RuntimeException("Interrupted while waiting for completion of original execution of the query. You can retry.");
            }
        }

        private void waitForUserLock() throws TimeoutException {
            if(waitLockForUser == null) {
                return;
            }

            try {
                if(!waitLockForUser.tryAcquire(5, TimeUnit.MINUTES)) {
                    throw new TimeoutException("Reached timeout (5 min) waiting in queue for query execution");
                }
                userSlotUsed = true;
            } catch (InterruptedException ignored) {
                throw new RuntimeException("Wait in queue for query execution was interrupted. You can retry.");
            }
        }

        @Override
        public void close() throws IOException {
            owner.release(this);
        }

        public void markAsynchronousRelease() {
            this.asynchronousRelease = true;
        }

        @JsonIgnore
        public boolean isAsynchronousRelease() {
            return asynchronousRelease;
        }
    }
}