DefaultProcessInstanceImpl.java

package io.cattle.platform.engine.process.impl;

import static io.cattle.platform.engine.process.ExitReason.*;
import static io.cattle.platform.util.time.TimeUtils.*;

import io.cattle.platform.archaius.util.ArchaiusUtil;
import io.cattle.platform.async.utils.TimeoutException;
import io.cattle.platform.deferred.util.DeferredUtils;
import io.cattle.platform.engine.context.EngineContext;
import io.cattle.platform.engine.eventing.EngineEvents;
import io.cattle.platform.engine.handler.HandlerResult;
import io.cattle.platform.engine.handler.ProcessHandler;
import io.cattle.platform.engine.handler.ProcessLogic;
import io.cattle.platform.engine.handler.ProcessPostListener;
import io.cattle.platform.engine.handler.ProcessPreListener;
import io.cattle.platform.engine.idempotent.Idempotent;
import io.cattle.platform.engine.idempotent.IdempotentExecution;
import io.cattle.platform.engine.idempotent.IdempotentRetryException;
import io.cattle.platform.engine.manager.ProcessManager;
import io.cattle.platform.engine.manager.impl.ProcessRecord;
import io.cattle.platform.engine.process.ExecutionExceptionHandler;
import io.cattle.platform.engine.process.ExitReason;
import io.cattle.platform.engine.process.LaunchConfiguration;
import io.cattle.platform.engine.process.Predicate;
import io.cattle.platform.engine.process.ProcessDefinition;
import io.cattle.platform.engine.process.ProcessInstance;
import io.cattle.platform.engine.process.ProcessInstanceException;
import io.cattle.platform.engine.process.ProcessPhase;
import io.cattle.platform.engine.process.ProcessResult;
import io.cattle.platform.engine.process.ProcessServiceContext;
import io.cattle.platform.engine.process.ProcessState;
import io.cattle.platform.engine.process.ProcessStateTransition;
import io.cattle.platform.engine.process.StateChangeMonitor;
import io.cattle.platform.engine.process.log.ExceptionLog;
import io.cattle.platform.engine.process.log.ParentLog;
import io.cattle.platform.engine.process.log.ProcessExecutionLog;
import io.cattle.platform.engine.process.log.ProcessLog;
import io.cattle.platform.engine.process.log.ProcessLogicExecutionLog;
import io.cattle.platform.eventing.model.Event;
import io.cattle.platform.eventing.model.EventVO;
import io.cattle.platform.lock.LockCallback;
import io.cattle.platform.lock.LockCallbackNoReturn;
import io.cattle.platform.lock.definition.DefaultMultiLockDefinition;
import io.cattle.platform.lock.definition.LockDefinition;
import io.cattle.platform.lock.definition.Namespace;
import io.cattle.platform.lock.exception.FailedToAcquireLockException;
import io.cattle.platform.util.exception.ExceptionUtils;
import io.cattle.platform.util.exception.ExecutionException;
import io.cattle.platform.util.exception.LoggableException;

import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Stack;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.netflix.config.DynamicLongProperty;

public class DefaultProcessInstanceImpl implements ProcessInstance {
    private static final DynamicLongProperty RETRY_MAX_WAIT = ArchaiusUtil.getLong("process.retry_max_wait.millis");
    private static final DynamicLongProperty RETRY_RUNNING_DELAY = ArchaiusUtil.getLong("process.running_delay.millis");

    private static final Logger log = LoggerFactory.getLogger(DefaultProcessInstanceImpl.class);

    ProcessServiceContext context;
    ProcessInstanceContext instanceContext;
    Stack<ProcessInstanceContext> instanceContextHistory = new Stack<ProcessInstanceContext>();

    ProcessRecord record;
    ProcessLog processLog;
    ProcessExecutionLog execution;
    ExitReason finalReason;
    String chainProcess;
    Date exceptionRunAfter;

    volatile boolean inLogic = false;
    boolean executed = false;
    boolean schedule = false;

    public DefaultProcessInstanceImpl(ProcessServiceContext context, ProcessRecord record, ProcessDefinition processDefinition, ProcessState state,
            boolean schedule, boolean replay) {
        super();
        this.schedule = schedule;
        this.context = context;
        this.instanceContext = new ProcessInstanceContext();
        this.instanceContext.setProcessDefinition(processDefinition);
        this.instanceContext.setState(state);
        this.instanceContext.setPhase(record.getPhase());
        this.instanceContext.setReplay(replay);
        this.record = record;

        this.processLog = record.getProcessLog();
    }

    @Override
    public ExitReason execute() {
        synchronized (this) {
            if (executed) {
                throw new IllegalStateException("A process can only be executed once");
            }
            executed = true;
        }

        if (record.getEndTime() != null) {
            return exit(ALREADY_DONE);
        }

        LockDefinition lockDef = new ProcessLock(this);
        try {
            log.debug("Attempting to run process [{}:{}] on resource [{}]", getName(), getId(), instanceContext.state.getResourceId());
            return context.getLockManager().lock(lockDef, new LockCallback<ExitReason>() {
                @Override
                public ExitReason doWithLock() {
                    return executeWithProcessInstanceLock();
                }
            });
        } catch (FailedToAcquireLockException e) {
            if (e.isLock(lockDef)) {
                exit(PROCESS_ALREADY_IN_PROGRESS);
                throw new ProcessInstanceException(this, new ProcessExecutionExitException(PROCESS_ALREADY_IN_PROGRESS));
            } else {
                throw e;
            }
        } finally {
            log.debug("Exiting [{}] process [{}:{}] on resource [{}]", finalReason, getName(), getId(), instanceContext.state.getResourceId());
        }
    }

    protected void openLog(EngineContext engineContext) {
        if (processLog == null) {
            ParentLog parentLog = engineContext.peekLog();
            if (parentLog == null) {
                processLog = new ProcessLog();
            } else {
                processLog = parentLog.newChildLog();
            }
        }

        execution = processLog.newExecution();
        execution.setProcessId(record.getId());
        execution.setResourceType(instanceContext.getProcessDefinition().getResourceType());
        execution.setResourceId(instanceContext.getState().getResourceId());
        execution.setProcessName(instanceContext.getProcessDefinition().getName());

        engineContext.pushLog(execution);
    }

    protected void closeLog(EngineContext engineContext) {
        engineContext.popLog();
    }

    protected ExitReason executeWithProcessInstanceLock() {
        EngineContext engineContext = EngineContext.getEngineContext();
        try {
            runDelegateLoop(engineContext);

            return exit(ExitReason.DONE);
        } catch (ProcessExecutionExitException e) {
            exit(e.getExitReason());
            if (e.getExitReason() != null && e.getExitReason().getResult() == ProcessResult.SUCCESS) {
                return e.getExitReason();
            }

            if (e.getExitReason().isRethrow()) {
                Throwable t = e.getCause();
                ExceptionUtils.rethrowRuntime(t == null ? e : t);
            }

            throw new ProcessInstanceException(this, e);
        } finally {
            context.getProcessManager().persistState(this, schedule);
        }
    }

    protected void runDelegateLoop(EngineContext engineContext) {
        while (true) {
            try {
                openLog(engineContext);

                preRunStateCheck();

                acquireLockAndRun();

                break;
            } catch (ProcessCancelException e) {
                if (shouldAbort(e)) {
                    if (!instanceContext.getState().shouldCancel(record) && instanceContext.getState().isTransitioning())
                        throw new IllegalStateException("Attempt to cancel when process is still transitioning", e);
                    throw e;
                } else {
                    execution.exit(DELEGATE);
                }
            } catch (ProcessExecutionExitException e) {
                e.log(log);
                throw e;
            } catch (IdempotentRetryException e) {
                execution.setException(new ExceptionLog(e));
                throw new ProcessExecutionExitException(RETRY_EXCEPTION, e);
            } catch (TimeoutException t) {
                throw new ProcessExecutionExitException(TIMEOUT, t);
            } catch (Throwable t) {
                execution.setException(new ExceptionLog(t));
                if (t instanceof LoggableException) {
                    ((LoggableException) t).log(log);
                } else {
                    log.error("Unknown exception", t);
                }
                throw new ProcessExecutionExitException(UNKNOWN_EXCEPTION, t);
            } finally {
                closeLog(engineContext);
            }
        }
    }

    protected boolean shouldAbort(ProcessCancelException e) {
        ProcessDefinition def = context.getProcessManager().getProcessDelegate(instanceContext.getProcessDefinition());
        if (def == null) {
            return true;
        }

        ProcessState state = def.constructProcessState(record);
        if (state.shouldCancel(record)) {
            return true;
        }

        ProcessInstanceContext newContext = new ProcessInstanceContext();
        newContext.setProcessDefinition(def);
        newContext.setState(state);
        newContext.setPhase(ProcessPhase.STARTED);

        instanceContextHistory.push(instanceContext);
        instanceContext = newContext;

        return false;
    }

    protected void acquireLockAndRun() {
        startLock();
        try {
            context.getLockManager().lock(instanceContext.getProcessLock(), new LockCallbackNoReturn() {
                @Override
                public void doWithLockNoResult() {
                    runWithProcessLock();
                }
            });
        } catch (FailedToAcquireLockException e) {
            throw new ProcessExecutionExitException(RESOURCE_BUSY, e);
        }
    }

    protected void preRunStateCheck() {
        if (instanceContext.getState().isDone(schedule)) {
            String configuredChainProcess = getConfiguredChainProcess();
            if (configuredChainProcess != null) {
                try {
                    scheduleChain(configuredChainProcess);
                    throw new ProcessExecutionExitException(ExitReason.CHAIN);
                } catch (ProcessCancelException e) {
                }
            }
            throw new ProcessExecutionExitException(ALREADY_DONE);
        }
    }

    protected boolean shouldCancel() {
        return instanceContext.getState().shouldCancel(record) ||
            (instanceContext.isReplay() && !instanceContext.getState().isTransitioning() && instanceContext.getState().isStart(record));
    }

    protected void incrementExecutionCountAndRunAfter() {
        long count = record.getExecutionCount();
        count += 1;

        record.setExecutionCount(count);
        record.setRunAfter(new Date(System.currentTimeMillis() + RETRY_RUNNING_DELAY.get()));
        record.setRunningProcessServerId(EngineContext.getProcessServerId());
    }

    protected void setNextRunAfter() {
        Long count = record.getExecutionCount();
        if (count == 0) {
            count = 1L;
        }

        long wait = RETRY_MAX_WAIT.get();
        double maxCount = Math.ceil(Math.log(RETRY_MAX_WAIT.get())/Math.log(2));
        if (count <= maxCount) {
            wait = Math.min(RETRY_MAX_WAIT.get(), Math.abs(2000L + (long)Math.pow(2, count-1) * 100));
        }
        record.setRunAfter(new Date(System.currentTimeMillis() + wait));
        if (exceptionRunAfter != null && exceptionRunAfter.after(record.getRunAfter())) {
            record.setRunAfter(exceptionRunAfter);
        }
        record.setRunningProcessServerId(null);
    }

    protected void runWithProcessLock() {
        String previousState = null;
        boolean success = false;
        try {
            ProcessState state = instanceContext.getState();
            state.rebuild();

            if (state.getResource() == null) {
                throw new ProcessCancelException("Resource is null [" + getName() + ":" + getId() + "] on resource [" +
                        getResourceId() + "]");
            }


            preRunStateCheck();

            if (shouldCancel()) {
                throw new ProcessCancelException("State [" + instanceContext.getState().getState() + "] is not valid for process [" +
                        getName() + ":" + getId() + "] on resource [" + getResourceId() + "]");
            }

            if (schedule) {
                Predicate predicate = record.getPredicate();
                if (predicate != null) {
                    if (!predicate.evaluate(this.instanceContext.getState(), this, this.instanceContext.getProcessDefinition())) {
                        throw new ProcessCancelException("Predicate is not valid");
                    }
                }
            }

            if (instanceContext.getState().isStart(record)) {
                previousState = setTransitioning();
            }

            if (schedule) {
                runScheduled();
            }

            if (instanceContext.getPhase() == ProcessPhase.REQUESTED) {
                instanceContext.setPhase(ProcessPhase.STARTED);
            }

            incrementExecutionCountAndRunAfter();
            getProcessManager().persistState(this, schedule);

            previousState = state.getState();

            runLogic();

            boolean chain = setDone(previousState);

            success = true;

            if (chain) {
                throw new ProcessExecutionExitException(ExitReason.CHAIN);
            }
        } catch (ProcessDelayException e) {
            exceptionRunAfter = e.getRunAfter();
            throw e;
        } finally {
            if (!schedule) {
                setNextRunAfter();
            }
            if (!success && !EngineContext.isNestedExecution()) {
                /*
                 * This is not so obvious why we do this. If a process fails it
                 * may have scheduled a compensating process. That means the
                 * state changed under the hood and we should look for that as
                 * it possibly cancels this process. If the process is nested,
                 * we don't want to cancel because it will mask the exception
                 * being thrown and additionally there is no process to cancel
                 * because the process is really owned by the parent. If we were
                 * to cancel a nested process, it will just look like a
                 * RuntimeException to the parent
                 */
                if (previousState != null)
                    assertState(previousState);
            }
        }
    }

    protected void runScheduled() {
        DeferredUtils.defer(new Runnable() {
            @Override
            public void run() {
                Long id = record.getId();
                if (id != null) {
                    Event event = EventVO.newEvent(EngineEvents.PROCESS_EXECUTE).withResourceId(id.toString());
                    context.getEventService().publish(event);
                }

            }
        });
        throw new ProcessExecutionExitException(SCHEDULED);
    }

    protected String getConfiguredChainProcess() {
        ProcessState state = instanceContext.getState();
        ProcessDefinition processDefinition = instanceContext.getProcessDefinition();

        if (state.getData().containsKey(processDefinition.getName() + ProcessLogic.CHAIN_PROCESS)) {
            return state.getData().get(processDefinition.getName() + ProcessLogic.CHAIN_PROCESS).toString();
        }

        return null;
    }

    protected boolean runHandlers(ProcessPhase phase, List<? extends ProcessLogic> handlers) {
        boolean shouldDelegate = false;
        final ProcessDefinition processDefinition = instanceContext.getProcessDefinition();
        final ProcessState state = instanceContext.getState();
        ProcessPhase currentPhase = instanceContext.getPhase();

        if (instanceContext.getPhase().ordinal() < phase.ordinal()) {
            final EngineContext context = EngineContext.getEngineContext();

            for (final ProcessLogic handler : handlers) {
                HandlerResult result = Idempotent.execute(new IdempotentExecution<HandlerResult>() {
                    @Override
                    public HandlerResult execute() {
                        if (Idempotent.enabled()) {
                            state.reload();
                        }
                        return runHandler(processDefinition, state, context, handler);
                    }
                });

                if (result != null) {
                    String chainResult = result.getChainProcessName();

                    if (chainResult != null) {
                        if (chainProcess != null && !chainResult.equals(chainProcess)) {
                            log.error("Not chaining process to [{}] because [{}] already set", chainResult, chainProcess);
                        }
                        chainProcess = chainResult;
                    }

                    shouldDelegate |= result.shouldDelegate();
                    if (!result.shouldContinue(instanceContext.getPhase())) {
                        break;
                    }
                }
            }

            String configuredChainProcess = getConfiguredChainProcess();
            if (currentPhase == ProcessPhase.POST_LISTENERS && chainProcess == null && configuredChainProcess != null ) {
                chainProcess = configuredChainProcess;
            }

            instanceContext.setPhase(phase);
        }

        return shouldDelegate;
    }

    protected String logicTypeString(ProcessLogic logic) {
        if (logic instanceof ProcessPreListener) {
            return "pre listener ";
        } else if (logic instanceof ProcessHandler) {
            return "handler ";
        } else if (logic instanceof ProcessPostListener) {
            return "post listener ";
        }

        return "";
    }

    protected HandlerResult runHandler(ProcessDefinition processDefinition, ProcessState state, EngineContext context, ProcessLogic handler) {
        final ProcessLogicExecutionLog processExecution = execution.newProcessLogicExecution(handler);
        context.pushLog(processExecution);
        try {
            log.debug("Running {}[{}]", logicTypeString(handler), handler.getName());
            HandlerResult handlerResult = handler.handle(state, DefaultProcessInstanceImpl.this);
            log.debug("Finished {}[{}]", logicTypeString(handler), handler.getName());

            if (handlerResult == null) {
                return handlerResult;
            }

            boolean shouldContinue = handlerResult.shouldContinue(state.getPhase());

            Map<String, Object> resultData = state.convertData(handlerResult.getData());

            processExecution.setShouldDelegate(handlerResult.shouldDelegate());
            processExecution.setShouldContinue(shouldContinue);
            processExecution.setChainProcessName(handlerResult.getChainProcessName());

            if (resultData.size() > 0) {
                state.applyData(resultData);
            }

            return handlerResult;
        } catch (ExecutionException e) {
            Idempotent.tempDisable();
            ExecutionExceptionHandler exceptionHandler = this.context.getExceptionHandler();
            if (exceptionHandler != null) {
                exceptionHandler.handleException(e, state, this.context);
            }
            processExecution.setException(new ExceptionLog(e));
            throw e;
        } catch (ProcessCancelException e) {
            throw e;
        } catch (RuntimeException e) {
            processExecution.setException(new ExceptionLog(e));
            throw e;
        } finally {
            processExecution.setStopTime(now());
            context.popLog();
        }
    }

    protected void runLogic() {
        inLogic = true;
        boolean shouldDelegate = false;
        try {
            instanceContext.setPhase(ProcessPhase.PRE_LISTENERS);
            shouldDelegate |= runHandlers(ProcessPhase.PRE_LISTENERS_DONE, instanceContext.getProcessDefinition().getPreProcessListeners());

            instanceContext.setPhase(ProcessPhase.HANDLERS);
            shouldDelegate |= runHandlers(ProcessPhase.HANDLER_DONE, instanceContext.getProcessDefinition().getProcessHandlers());

            instanceContext.setPhase(ProcessPhase.POST_LISTENERS);
            shouldDelegate |= runHandlers(ProcessPhase.POST_LISTENERS_DONE, instanceContext.getProcessDefinition().getPostProcessListeners());

            if (shouldDelegate) {
                throw new ProcessCancelException("Process result triggered a delegation");
            }
        } finally {
            inLogic = false;
        }

        instanceContext.setPhase(ProcessPhase.DONE);
    }

    protected void assertState(String previousState) {
        ProcessState state = instanceContext.getState();

        state.reload();
        String newState = state.getState();
        if (!previousState.equals(newState)) {
            preRunStateCheck();
            throw new ProcessExecutionExitException("Previous state [" + previousState + "] does not equal current state [" + newState + "]", STATE_CHANGED);
        }
    }

    public ProcessRecord getProcessRecord() {
        record.setPhase(instanceContext.getPhase());
        record.setProcessLog(processLog);
        record.setExitReason(finalReason);

        if (finalReason == null) {
            record.setRunningProcessServerId(EngineContext.getProcessServerId());
        } else {
            if (finalReason == ExitReason.RETRY_EXCEPTION) {
                record.setRunAfter(null);
            }
            record.setRunningProcessServerId(null);
            if (finalReason.isTerminating() && execution != null && execution.getStopTime() != null) {
                record.setEndTime(new Date(execution.getStopTime()));
            }

            record.setResult(finalReason.getResult());
        }

        return record;
    }

    protected String setTransitioning() {
        ProcessState state = instanceContext.getState();

        String previousState = state.getState();
        String newState = state.setTransitioning();
        log.debug("Changing state [{}->{}] on [{}:{}]", previousState, newState, record.getResourceType(), record.getResourceId());
        execution.getTransitions().add(new ProcessStateTransition(previousState, newState, "transitioning", now()));
        publishChanged(previousState, newState, schedule);

        return newState;
    }

    protected void scheduleChain(final String chainProcess) {
        final ProcessState state = instanceContext.getState();
        final LaunchConfiguration config = new LaunchConfiguration(chainProcess, record.getResourceType(), record.getResourceId(), record.getAccountId(),
                record.getPriority(), state.getData());
        config.setParentProcessState(state);

        ExecutionExceptionHandler handler = this.context.getExceptionHandler();

        Runnable run = new Runnable() {
            @Override
            public void run() {
                DefaultProcessInstanceImpl.this.context.getProcessManager().scheduleProcessInstance(config);
                log.debug("Chained [{}] to [{}]", record.getProcessName(), chainProcess);
                state.reload();
            }
        };

        if (handler == null) {
            run.run();
        } else {
            handler.wrapChainSchedule(state, context, run);
        }
    }

    protected boolean setDone(String previousState) {
        boolean chained = false;
        final ProcessState state = instanceContext.getState();
        assertState(previousState);

        if (chainProcess != null) {
            scheduleChain(chainProcess);
            chained = true;
        }

        String newState = chained ? state.getState() : state.setDone();
        log.debug("Changing state [{}->{}] on [{}:{}]", previousState, newState, record.getResourceType(), record.getResourceId());
        execution.getTransitions().add(new ProcessStateTransition(previousState, newState, chained ? "chain" : "done", now()));
        publishChanged(previousState, newState, schedule);

        return chained;
    }

    protected void publishChanged(String previousState, String newState, boolean defer) {
        for (StateChangeMonitor monitor : context.getChangeMonitors()) {
            monitor.onChange(defer, previousState, newState, record, instanceContext.getState(), context);
        }
    }

    protected void startLock() {
        ProcessState state = instanceContext.getState();

        LockDefinition lockDef = state.getProcessLock();
        if (schedule) {
            LockDefinition scheduleLock = new Namespace("schedule").getLockDefinition(lockDef);
            if (record.getPredicate() == null) {
                lockDef = scheduleLock;
            } else {
                lockDef = new DefaultMultiLockDefinition(lockDef, scheduleLock);
            }
        }

        instanceContext.setProcessLock(lockDef);
    }

    protected ExitReason exit(ExitReason reason) {
        if (execution != null) {
            execution.exit(reason);
        }

        return finalReason = reason;
    }

    @Override
    public Long getId() {
        return record.getId();
    }

    @Override
    public String getName() {
        return instanceContext.getProcessDefinition().getName();
    }

    @Override
    public boolean isRunningLogic() {
        return inLogic;
    }

    @Override
    public ExitReason getExitReason() {
        return finalReason;
    }

    @Override
    public ProcessManager getProcessManager() {
        return context.getProcessManager();
    }

    @Override
    public String toString() {
        try {
            return "Process [" + instanceContext.getProcessDefinition().getName() + "] resource [" + instanceContext.getState().getResourceId() + "]";
        } catch (NullPointerException e) {
            return super.toString();
        }
    }

    @Override
    public String getResourceId() {
        return instanceContext.getState().getResourceId();
    }

}