PerfTestRunnable.java

/* 
 * Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License. 
 */
package org.ngrinder.perftest.service;

import net.grinder.SingleConsole;
import net.grinder.SingleConsole.ConsoleShutdownListener;
import net.grinder.StopReason;
import net.grinder.common.GrinderProperties;
import net.grinder.console.model.ConsoleProperties;
import net.grinder.util.ListenerHelper;
import net.grinder.util.ListenerSupport;
import net.grinder.util.UnitUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.time.DateUtils;
import org.ngrinder.common.constant.ControllerConstants;
import org.ngrinder.extension.OnTestLifeCycleRunnable;
import org.ngrinder.extension.OnTestSamplingRunnable;
import org.ngrinder.infra.config.Config;
import org.ngrinder.infra.plugin.PluginManager;
import org.ngrinder.infra.schedule.ScheduledTaskService;
import org.ngrinder.model.PerfTest;
import org.ngrinder.model.Status;
import org.ngrinder.perftest.model.NullSingleConsole;
import org.ngrinder.perftest.service.samplinglistener.*;
import org.ngrinder.script.handler.ScriptHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import static org.apache.commons.lang.ObjectUtils.defaultIfNull;
import static org.ngrinder.common.constant.ClusterConstants.PROP_CLUSTER_SAFE_DIST;
import static org.ngrinder.common.util.AccessUtils.getSafe;
import static org.ngrinder.model.Status.*;

/**
 * {@link PerfTest} run scheduler.
 * <p/>
 * This class is responsible to execute/finish the performance test. The job is
 * started from {@link #doStart()}  and {@link #doFinish()} method. These
 * methods are scheduled by Spring Task.
 *
 * @author JunHo Yoon
 * @since 3.0
 */
@Profile("production")
@Component
public class PerfTestRunnable implements ControllerConstants {

	private static final Logger LOG = LoggerFactory.getLogger(PerfTestRunnable.class);

	@SuppressWarnings("SpringJavaAutowiringInspection")
	@Autowired
	private PerfTestService perfTestService;

	@Autowired
	private ConsoleManager consoleManager;

	@Autowired
	private AgentManager agentManager;

	@Autowired
	private PluginManager pluginManager;

	@Autowired
	private Config config;

	@Autowired
	private ScheduledTaskService scheduledTaskService;

	private Runnable startRunnable;

	private Runnable finishRunnable;

	@PostConstruct
	public void init() {
		// Clean up db first.
		doFinish(true);

		this.startRunnable = new Runnable() {
			@Override
			public void run() {
				startPeriodically();
			}
		};
		scheduledTaskService.addFixedDelayedScheduledTask(startRunnable, PERFTEST_RUN_FREQUENCY_MILLISECONDS);
		this.finishRunnable = new Runnable() {
			@Override
			public void run() {
				finishPeriodically();
			}
		};
		scheduledTaskService.addFixedDelayedScheduledTask(finishRunnable, PERFTEST_RUN_FREQUENCY_MILLISECONDS);

	}

	@PreDestroy
	public void destroy() {
		scheduledTaskService.removeScheduledJob(this.startRunnable);
		scheduledTaskService.removeScheduledJob(this.finishRunnable);
	}

	/**
	 * Scheduled method for test execution. This method dispatches the test
	 * candidates and run one of them. This method is responsible until a test
	 * is executed.
	 */
	public void startPeriodically() {
		doStart();
	}

	void doStart() {
		if (config.hasNoMoreTestLock()) {
			return;
		}
		// Block if the count of testing exceed the limit
		if (!canExecuteMore()) {
			// LOG MORE
			List<PerfTest> currentlyRunningTests = perfTestService.getCurrentlyRunningTest();
			LOG.debug("Currently running test is {}. No more tests can not run.", currentlyRunningTests.size());
			return;
		}
		// Find out next ready perftest
		PerfTest runCandidate = getRunnablePerfTest();
		if (runCandidate == null) {
			return;
		}

		if (!isScheduledNow(runCandidate)) {
			// this test project is reserved,but it isn't yet going to run test
			// right now.
			return;
		}


		if (!hasEnoughFreeAgents(runCandidate)) {
			return;
		}

		doTest(runCandidate);
	}

	private PerfTest getRunnablePerfTest() {
		return perfTestService.getNextRunnablePerfTestPerfTestCandidate();
	}

	private boolean canExecuteMore() {
		return consoleManager.getConsoleInUse().size() < perfTestService.getMaximumConcurrentTestCount();
	}

	private boolean isScheduledNow(PerfTest test) {
		Date current = new Date();
		Date scheduledDate = DateUtils
				.truncate((Date) defaultIfNull(test.getScheduledTime(), current), Calendar.MINUTE);
		return current.after(scheduledDate);
	}


	/**
	 * Check the free agent availability for the given {@link PerfTest}.
	 *
	 * @param test {@link PerfTest}
	 * @return true if enough agents
	 */
	protected boolean hasEnoughFreeAgents(PerfTest test) {
		int size = agentManager.getAllFreeApprovedAgentsForUser(test.getCreatedUser()).size();
		if (test.getAgentCount() != null && test.getAgentCount() > size) {
			perfTestService.markProgress(test, "The test is tried to execute but there is not enough free agents."
					+ "\n- Current free agent count : " + size + "  / Requested : " + test.getAgentCount() + "\n");
			return false;
		}
		return true;
	}

	/**
	 * Run the given test.
	 * <p/>
	 * If fails, it marks STOP_BY_ERROR in the given {@link PerfTest} status
	 *
	 * @param perfTest perftest instance;
	 */
	public void doTest(final PerfTest perfTest) {
		SingleConsole singleConsole = null;
		try {
			singleConsole = startConsole(perfTest);
			ScriptHandler prepareDistribution = perfTestService.prepareDistribution(perfTest);
			GrinderProperties grinderProperties = perfTestService.getGrinderProperties(perfTest, prepareDistribution);
			startAgentsOn(perfTest, grinderProperties, checkCancellation(singleConsole));
			distributeFileOn(perfTest, checkCancellation(singleConsole));

			singleConsole.setReportPath(perfTestService.getReportFileDirectory(perfTest));
			runTestOn(perfTest, grinderProperties, checkCancellation(singleConsole));
		} catch (SingleConsoleCancellationException ex) {
			// In case of error, mark the occurs error on perftest.
			doCancel(perfTest, singleConsole);
			notifyFinish(perfTest, StopReason.CANCEL_BY_USER);
		} catch (Exception e) {
			// In case of error, mark the occurs error on perftest.
			LOG.error("Error while executing test: {} - {} ", perfTest.getTestIdentifier(), e.getMessage());
			LOG.debug("Stack Trace is : ", e);
			doTerminate(perfTest, singleConsole);
			notifyFinish(perfTest, StopReason.ERROR_WHILE_PREPARE);
		}
	}

	/**
	 * Check the cancellation status on console.
	 *
	 * @param singleConsole console
	 * @return true if cancellation is requested.
	 */
	SingleConsole checkCancellation(SingleConsole singleConsole) {
		if (singleConsole.isCanceled()) {
			throw new SingleConsoleCancellationException("Single Console " + singleConsole.getConsolePort()
					+ " is canceled");
		}
		return singleConsole;
	}

	/**
	 * Start a console for given {@link PerfTest}.
	 *
	 * @param perfTest perftest
	 * @return started console
	 */
	SingleConsole startConsole(PerfTest perfTest) {
		perfTestService.markStatusAndProgress(perfTest, START_CONSOLE, "Console is being prepared.");
		// get available consoles.
		ConsoleProperties consoleProperty = perfTestService.createConsoleProperties(perfTest);
		SingleConsole singleConsole = consoleManager.getAvailableConsole(consoleProperty);
		singleConsole.start();
		perfTestService.markPerfTestConsoleStart(perfTest, singleConsole.getConsolePort());
		return singleConsole;
	}

	/**
	 * Distribute files to agents.
	 *
	 * @param perfTest      perftest
	 * @param singleConsole console to be used.
	 */
	void distributeFileOn(final PerfTest perfTest, SingleConsole singleConsole) {
		// Distribute files
		perfTestService.markStatusAndProgress(perfTest, DISTRIBUTE_FILES, "All necessary files are being distributed.");
		ListenerSupport<SingleConsole.FileDistributionListener> listener = ListenerHelper.create();
		final long safeThreadHold = getSafeTransmissionThreshold();

		listener.add(new SingleConsole.FileDistributionListener() {
			@Override
			public void distributed(String fileName) {
				perfTestService.markProgress(perfTest, " - " + fileName);
			}

			@Override
			public boolean start(File dir, boolean safe) {
				if (safe) {
					perfTestService.markProgress(perfTest, "Safe file distribution mode is enabled.");
					return safe;
				}
				long sizeOfDirectory = FileUtils.sizeOfDirectory(dir);
				if (sizeOfDirectory > safeThreadHold) {
					perfTestService.markProgress(perfTest, "The total size of distributed files is over "
							+ UnitUtils.byteCountToDisplaySize(safeThreadHold) + "B.\n- Safe file distribution mode is enabled by force.");
					return true;
				}
				return safe;
			}

		});

		// the files have prepared before
		singleConsole.distributeFiles(perfTestService.getDistributionPath(perfTest), listener,
				isSafeDistPerfTest(perfTest));
		perfTestService.markStatusAndProgress(perfTest, DISTRIBUTE_FILES_FINISHED,
				"All necessary files are distributed.");
	}

	protected long getSafeTransmissionThreshold() {
		return config.getControllerProperties().getPropertyLong(PROP_CONTROLLER_SAFE_DIST_THRESHOLD);
	}

	private boolean isSafeDistPerfTest(final PerfTest perfTest) {
		boolean safeDist = getSafe(perfTest.getSafeDistribution());
		if (config.isClustered()) {
			safeDist = config.getClusterProperties().getPropertyBoolean(PROP_CLUSTER_SAFE_DIST);
		}
		return safeDist;
	}

	/**
	 * Start agents for the given {@link PerfTest}.
	 *
	 * @param perfTest          perftest
	 * @param grinderProperties grinder properties
	 * @param singleConsole     console to be used.
	 */
	void startAgentsOn(PerfTest perfTest, GrinderProperties grinderProperties, SingleConsole singleConsole) {
		perfTestService.markStatusAndProgress(perfTest, START_AGENTS, getSafe(perfTest.getAgentCount())
				+ " agents are starting.");
		agentManager.runAgent(perfTest.getCreatedUser(), singleConsole, grinderProperties,
				getSafe(perfTest.getAgentCount()));
		singleConsole.waitUntilAgentConnected(perfTest.getAgentCount());
		perfTestService.markStatusAndProgress(perfTest, START_AGENTS_FINISHED, getSafe(perfTest.getAgentCount())
				+ " agents are ready.");
	}

	/**
	 * Run a given {@link PerfTest} with the given {@link GrinderProperties} and
	 * the {@link SingleConsole} .
	 *
	 * @param perfTest          perftest
	 * @param grinderProperties grinder properties
	 * @param singleConsole     console to be used.
	 */
	void runTestOn(final PerfTest perfTest, GrinderProperties grinderProperties, final SingleConsole singleConsole) {
		// start target monitor
		for (OnTestLifeCycleRunnable run : pluginManager.getEnabledModulesByClass(OnTestLifeCycleRunnable.class)) {
			run.start(perfTest, perfTestService, config.getVersion());
		}

		// Run test
		perfTestService.markStatusAndProgress(perfTest, START_TESTING, "The test is ready to start.");
		// Add listener to detect abnormal condition and mark the perfTest
		singleConsole.addListener(new ConsoleShutdownListener() {
			@Override
			public void readyToStop(StopReason stopReason) {
				perfTestService.markAbnormalTermination(perfTest, stopReason);
				LOG.error("Abnormal test {} due to {}", perfTest.getId(), stopReason.name());
			}
		});
		long startTime = singleConsole.startTest(grinderProperties);
		perfTest.setStartTime(new Date(startTime));
		addSamplingListeners(perfTest, singleConsole);
		perfTestService.markStatusAndProgress(perfTest, TESTING, "The test is started.");
		singleConsole.startSampling();

	}

	protected void addSamplingListeners(final PerfTest perfTest, final SingleConsole singleConsole) {
		// Add SamplingLifeCycleListener
		singleConsole.addSamplingLifeCyleListener(new PerfTestSamplingCollectorListener(singleConsole,
				perfTest.getId(), perfTestService, scheduledTaskService));
		singleConsole.addSamplingLifeCyleListener(new AgentLostDetectionListener(singleConsole, perfTest,
				perfTestService, scheduledTaskService));
		List<OnTestSamplingRunnable> testSamplingPlugins = pluginManager.getEnabledModulesByClass
				(OnTestSamplingRunnable.class, new MonitorCollectorPlugin(config, scheduledTaskService,
						perfTestService, perfTest.getId()));
		singleConsole.addSamplingLifeCyleListener(new PluginRunListener(testSamplingPlugins, singleConsole,
				perfTest, perfTestService));
		singleConsole.addSamplingLifeCyleListener(new AgentDieHardListener(singleConsole, perfTest, perfTestService,
				agentManager, scheduledTaskService));
	}


	/**
	 * Notify test finish to plugins.
	 *
	 * @param perfTest PerfTest
	 * @param reason   the reason of test finish..
	 * @see OnTestLifeCycleRunnable
	 */
	public void notifyFinish(PerfTest perfTest, StopReason reason) {
		for (OnTestLifeCycleRunnable run : pluginManager.getEnabledModulesByClass(OnTestLifeCycleRunnable.class)) {
			run.finish(perfTest, reason.name(), perfTestService, config.getVersion());
		}
	}

	/**
	 * Finish the tests.(Scheduled by SpringTask)
	 * <p/>
	 * There are three types of test finish.
	 * <p/>
	 * <ul>
	 * <li>Abnormal test finish : when TPS is too low or too many errors occur</li>
	 * <li>User requested test finish : when user requested to finish the test</li>
	 * <li>Normal test finish : when the test reaches the planned duration and run
	 * count.</li>
	 * </ul>
	 */
	public void finishPeriodically() {
		doFinish(false);
	}

	protected void doFinish(boolean initial) {
		if (!initial && consoleManager.getConsoleInUse().isEmpty()) {
			return;
		}
		doFinish();
	}

	void doFinish() {
		for (PerfTest each : perfTestService.getAllAbnormalTesting()) {
			LOG.info("Terminate {}", each.getId());
			SingleConsole consoleUsingPort = consoleManager.getConsoleUsingPort(each.getPort());
			doTerminate(each, consoleUsingPort);
			cleanUp(each);
			notifyFinish(each, StopReason.TOO_MANY_ERRORS);
		}

		for (PerfTest each : perfTestService.getAllStopRequested()) {
			LOG.info("Stop test {}", each.getId());
			SingleConsole consoleUsingPort = consoleManager.getConsoleUsingPort(each.getPort());
			doCancel(each, consoleUsingPort);
			cleanUp(each);
			notifyFinish(each, StopReason.CANCEL_BY_USER);
		}

		for (PerfTest each : perfTestService.getAllTesting()) {
			SingleConsole consoleUsingPort = consoleManager.getConsoleUsingPort(each.getPort());
			if (isTestFinishCandidate(each, consoleUsingPort)) {
				doNormalFinish(each, consoleUsingPort);
				cleanUp(each);
				notifyFinish(each, StopReason.NORMAL);
			}
		}
	}

	/**
	 * Clean up distribution directory for the given perfTest.
	 *
	 * @param perfTest perfTest
	 */
	private void cleanUp(PerfTest perfTest) {
		perfTestService.cleanUpDistFolder(perfTest);
		perfTestService.cleanUpRuntimeOnlyData(perfTest);
	}

	/**
	 * Check if the given {@link PerfTest} is ready to finish.
	 *
	 * @param perfTest           perf test
	 * @param singleConsoleInUse singleConsole
	 * @return true if it's a finish candidate.
	 */
	private boolean isTestFinishCandidate(PerfTest perfTest, SingleConsole singleConsoleInUse) {
		// Give 5 seconds to be finished
		if (perfTest.isThresholdDuration()
				&& singleConsoleInUse.isCurrentRunningTimeOverDuration(perfTest.getDuration())) {
			LOG.debug(
					"Test {} is ready to finish. Current : {}, Planned : {}",
					new Object[]{perfTest.getTestIdentifier(), singleConsoleInUse.getCurrentRunningTime(),
							perfTest.getDuration()});
			return true;
		} else if (perfTest.isThresholdRunCount()
				&& singleConsoleInUse.getCurrentExecutionCount() >= perfTest.getTotalRunCount()) {
			LOG.debug("Test {} is ready to finish. Current : {}, Planned : {}",
					new Object[]{perfTest.getTestIdentifier(), singleConsoleInUse.getCurrentExecutionCount(),
							perfTest.getTotalRunCount()});
			return true;
		} else if (singleConsoleInUse instanceof NullSingleConsole) {
			LOG.debug("Test {} is ready to finish. Current : {}, Planned : {}",
					new Object[]{perfTest.getTestIdentifier(), singleConsoleInUse.getCurrentExecutionCount(),
							perfTest.getTotalRunCount()});
			return true;
		}

		return false;
	}

	/**
	 * Cancel the given {@link PerfTest}.
	 *
	 * @param perfTest           {@link PerfTest} to be canceled.
	 * @param singleConsoleInUse {@link SingleConsole} which is being used for the given
	 *                           {@link PerfTest}
	 */
	public void doCancel(PerfTest perfTest, SingleConsole singleConsoleInUse) {
		LOG.info("Cancel test {} by user request.", perfTest.getId());
		singleConsoleInUse.unregisterSampling();
		try {
			perfTestService.markProgressAndStatusAndFinishTimeAndStatistics(perfTest, CANCELED,
					"Stop requested by user");
		} catch (Exception e) {
			LOG.error("Error while canceling test {} : {}", perfTest.getId(), e.getMessage());
			LOG.debug("Details : ", e);
		}
		consoleManager.returnBackConsole(perfTest.getTestIdentifier(), singleConsoleInUse);
	}

	/**
	 * Terminate the given {@link PerfTest}.
	 *
	 * @param perfTest           {@link PerfTest} to be finished
	 * @param singleConsoleInUse {@link SingleConsole} which is being used for the given
	 *                           {@link PerfTest}
	 */
	public void doTerminate(PerfTest perfTest, SingleConsole singleConsoleInUse) {
		singleConsoleInUse.unregisterSampling();
		try {
			perfTestService.markProgressAndStatusAndFinishTimeAndStatistics(perfTest, Status.STOP_BY_ERROR,
					"Stopped by error");
		} catch (Exception e) {
			LOG.error("Error while terminating {} : {}", perfTest.getTestIdentifier(), e.getMessage());
			LOG.debug("Details : ", e);
		}
		consoleManager.returnBackConsole(perfTest.getTestIdentifier(), singleConsoleInUse);
	}

	/**
	 * Finish the given {@link PerfTest}.
	 *
	 * @param perfTest           {@link PerfTest} to be finished
	 * @param singleConsoleInUse {@link SingleConsole} which is being used for the given
	 *                           {@link PerfTest}
	 */
	public void doNormalFinish(PerfTest perfTest, SingleConsole singleConsoleInUse) {
		LOG.debug("PerfTest {} status - currentRunningTime {} ", perfTest.getId(),
				singleConsoleInUse.getCurrentRunningTime());
		singleConsoleInUse.unregisterSampling();
		try {
			// stop target host monitor
			if (perfTestService.hasTooManyError(perfTest)) {
				perfTestService.markProgressAndStatusAndFinishTimeAndStatistics(perfTest, Status.STOP_BY_ERROR,
						"[WARNING] The test is finished but contains too much errors(over 30% of total runs).");
			} else if (singleConsoleInUse.hasNoPerformedTest()) {
				perfTestService.markProgressAndStatusAndFinishTimeAndStatistics(perfTest, Status.STOP_BY_ERROR,
						"[WARNING] The test is finished but has no TPS.");
			} else {
				perfTestService.markProgressAndStatusAndFinishTimeAndStatistics(perfTest, Status.FINISHED,
						"The test is successfully finished.");
			}
		} catch (Exception e) {
			perfTestService.markStatusAndProgress(perfTest, Status.STOP_BY_ERROR, e.getMessage());
			LOG.error("Error while finishing {} : {}", perfTest.getTestIdentifier(), e.getMessage());
			LOG.debug("Details : ", e);
		}
		consoleManager.returnBackConsole(perfTest.getTestIdentifier(), singleConsoleInUse);
	}

	public PerfTestService getPerfTestService() {
		return perfTestService;
	}

	public AgentManager getAgentManager() {
		return agentManager;
	}

}