/*
* ProActive Parallel Suite(TM):
* The Open Source library for parallel and distributed
* Workflows & Scheduling, Orchestration, Cloud Automation
* and Big Data Analysis on Enterprise Grids & Clouds.
*
* Copyright (c) 2007 - 2017 ActiveEon
* Contact: contact@activeeon.com
*
* This library is free software: you can redistribute it and/or
* modify it under the terms of the GNU Affero General Public License
* as published by the Free Software Foundation: version 3 of
* the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* If needed, contact us to obtain a release under GPL Version 2 or 3
* or a different license than the AGPL.
*/
package org.ow2.proactive.scheduler.core.db.schedulerdb;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.hibernate.Session;
import org.hibernate.metadata.ClassMetadata;
import org.junit.Assert;
import org.junit.Test;
import org.ow2.proactive.scheduler.common.job.JobStatus;
import org.ow2.proactive.scheduler.common.job.TaskFlowJob;
import org.ow2.proactive.scheduler.common.task.ForkEnvironment;
import org.ow2.proactive.scheduler.common.task.JavaTask;
import org.ow2.proactive.scheduler.common.task.NativeTask;
import org.ow2.proactive.scheduler.common.task.Task;
import org.ow2.proactive.scheduler.common.task.dataspaces.InputAccessMode;
import org.ow2.proactive.scheduler.common.task.dataspaces.OutputAccessMode;
import org.ow2.proactive.scheduler.common.task.flow.FlowScript;
import org.ow2.proactive.scheduler.core.db.JobContent;
import org.ow2.proactive.scheduler.core.db.JobData;
import org.ow2.proactive.scheduler.core.db.TaskData;
import org.ow2.proactive.scheduler.core.db.TaskResultData;
import org.ow2.proactive.scheduler.job.InternalJob;
import org.ow2.proactive.scheduler.task.TaskResultImpl;
import org.ow2.proactive.scripting.SelectionScript;
import org.ow2.proactive.scripting.SimpleScript;
import com.google.common.collect.ImmutableSet;
public class TestJobRemove extends BaseSchedulerDBTest {
@Test
public void testRuntimeDataRemoveAfterFinish() throws Throwable {
TaskFlowJob jobDef = createJob(2);
InternalJob job = defaultSubmitJobAndLoadInternal(false, jobDef);
dbManager.updateAfterTaskFinished(job, job.getTask("javaTask-0"), new TaskResultImpl(null, "OK1", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("forkedJavaTask-0"),
new TaskResultImpl(null, "OK2", null, 0));
dbManager.updateAfterTaskFinished(job, job.getTask("nativeTask-0"), new TaskResultImpl(null, "OK3", null, 0));
job.setStatus(JobStatus.FINISHED);
dbManager.updateAfterTaskFinished(job, null, null);
checkAllEntitiesDeleted(JobData.class.getName(),
JobContent.class.getName(),
TaskData.class.getName(),
TaskResultData.class.getName());
// check can still load task results
Assert.assertEquals("OK1", dbManager.loadTaskResult(job.getTask("javaTask-0").getId(), 0).value());
Assert.assertEquals("OK2", dbManager.loadTaskResult(job.getTask("forkedJavaTask-0").getId(), 0).value());
Assert.assertEquals("OK3", dbManager.loadTaskResult(job.getTask("nativeTask-0").getId(), 0).value());
}
@Test
public void testRuntimeDataRemoveAfterFinishParallel() throws Throwable {
int THREAD_COUNT = 4;
ExecutorService executorService = Executors.newCachedThreadPool();
List<InternalJob> jobs = new ArrayList<>(THREAD_COUNT);
TaskFlowJob jobDef;
for (int i = 0; i < THREAD_COUNT; i++) {
jobDef = createJob(2);
jobs.add(defaultSubmitJobAndLoadInternal(false, jobDef));
}
for (int i = 0; i < THREAD_COUNT; i++) {
final InternalJob job = jobs.get(i);
executorService.submit(new Runnable() {
@Override
public void run() {
try {
dbManager.updateAfterTaskFinished(job,
job.getTask("javaTask-0"),
new TaskResultImpl(null, "OK1", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("forkedJavaTask-0"),
new TaskResultImpl(null, "OK2", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("nativeTask-0"),
new TaskResultImpl(null, "OK3", null, 0));
job.setStatus(JobStatus.FINISHED);
dbManager.updateAfterTaskFinished(job, null, null);
} catch (Throwable throwable) {
throwable.printStackTrace();
}
}
});
}
executorService.shutdown();
executorService.awaitTermination(30, TimeUnit.SECONDS);
for (int i = 0; i < THREAD_COUNT; i++) {
final InternalJob job = jobs.get(i);
List<InternalJob> jobsNotFinished = dbManager.loadNotFinishedJobs(true);
Assert.assertEquals("All jobs should be finished", 0, jobsNotFinished.size());
checkAllEntitiesDeleted(JobData.class.getName(),
JobContent.class.getName(),
TaskData.class.getName(),
TaskResultData.class.getName());
// check can still load task results
Assert.assertEquals("OK1", dbManager.loadTaskResult(job.getTask("javaTask-0").getId(), 0).value());
Assert.assertEquals("OK2", dbManager.loadTaskResult(job.getTask("forkedJavaTask-0").getId(), 0).value());
Assert.assertEquals("OK3", dbManager.loadTaskResult(job.getTask("nativeTask-0").getId(), 0).value());
}
}
@Test
public void testSetRemovedTime() throws Exception {
InternalJob job = defaultSubmitJobAndLoadInternal(false, new TaskFlowJob());
Assert.assertEquals(-1L, job.getRemovedTime());
long removedTime = System.currentTimeMillis();
dbManager.removeJob(job.getId(), removedTime, false);
job = loadInternalJob(false, job.getId());
Assert.assertEquals(removedTime, job.getRemovedTime());
}
@Test
public void testFullDataRemove1() throws Exception {
TaskFlowJob jobDef = new TaskFlowJob();
JavaTask task1 = new JavaTask();
task1.setName("task1");
task1.setExecutableClassName(TestDummyExecutable.class.getName());
jobDef.addTask(task1);
InternalJob job = defaultSubmitJob(jobDef);
System.out.println("Remove job");
long start = System.currentTimeMillis();
dbManager.removeJob(job.getId(), 0, true);
System.out.println("Remove time (single task)" + (System.currentTimeMillis() - start));
checkAllEntitiesDeleted();
}
@Test
public void testFullDataRemove2() throws Exception {
removeScenario(1);
}
@Test
public void testFullDataRemoveManyTasks() throws Exception {
removeScenario(100);
}
private TaskFlowJob createJob(int tasksNumber) throws Exception {
ForkEnvironment forkEnvironment = new ForkEnvironment();
forkEnvironment.addAdditionalClasspath("lib/ProActive/ProActive.jar", "compile/lib/ant.jar");
TaskFlowJob jobDef = new TaskFlowJob();
jobDef.addGenericInformation("k1", "v1");
jobDef.addGenericInformation("k2", "v2");
// add data with non-null ifBranch
JavaTask A = createDefaultTask("A");
A.setForkEnvironment(forkEnvironment);
FlowScript ifScript = FlowScript.createIfFlowScript("branch = \"if\";", "B", "C", null);
A.setFlowScript(ifScript);
jobDef.addTask(A);
JavaTask B = createDefaultTask("B");
B.setForkEnvironment(forkEnvironment);
jobDef.addTask(B);
JavaTask C = createDefaultTask("C");
C.setForkEnvironment(forkEnvironment);
jobDef.addTask(C);
for (int i = 0; i < tasksNumber; i++) {
JavaTask task1 = new JavaTask();
task1.setName("javaTask-" + i);
task1.setExecutableClassName(TestDummyExecutable.class.getName());
task1.addArgument("arg1", "arg1");
task1.addArgument("arg2", "arg2");
setAttributes(task1);
JavaTask task2 = new JavaTask();
task2.setName("forkedJavaTask-" + i);
task2.setExecutableClassName(TestDummyExecutable.class.getName());
ForkEnvironment forkEnv = new ForkEnvironment();
forkEnv.addAdditionalClasspath("lib/ProActive/ProActive.jar");
forkEnv.addAdditionalClasspath("compile/lib/ant.jar");
forkEnv.addJVMArgument("jvmArg1");
forkEnv.addJVMArgument("jvmArg2");
forkEnv.addSystemEnvironmentVariable("e1", "v1");
forkEnv.addSystemEnvironmentVariable("e2", "v2");
forkEnv.setEnvScript(new SimpleScript("env script", "javascript", new String[] { "param1", "param2" }));
task2.setForkEnvironment(forkEnv);
task2.addArgument("arg1", "arg1");
task2.addArgument("arg2", "arg2");
setAttributes(task2);
NativeTask task3 = new NativeTask();
task3.setName("nativeTask-" + i);
task3.setCommandLine("command1", "command2", "command3");
setAttributes(task3);
task1.addDependence(task2);
task3.addDependence(task2);
task1.setForkEnvironment(forkEnvironment);
task2.setForkEnvironment(forkEnvironment);
task3.setForkEnvironment(forkEnvironment);
jobDef.addTask(task1);
jobDef.addTask(task2);
jobDef.addTask(task3);
}
return jobDef;
}
private void removeScenario(int tasksNumber) throws Exception {
TaskFlowJob jobDef = createJob(tasksNumber);
InternalJob job = defaultSubmitJobAndLoadInternal(false, jobDef);
for (int i = 0; i < tasksNumber; i++) {
dbManager.updateAfterTaskFinished(job,
job.getTask("javaTask-" + i),
new TaskResultImpl(null, "OK", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("javaTask-" + i),
new TaskResultImpl(null, "OK", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("forkedJavaTask-" + i),
new TaskResultImpl(null, "OK", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("forkedJavaTask-" + i),
new TaskResultImpl(null, "OK", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("nativeTask-" + i),
new TaskResultImpl(null, "OK", null, 0));
dbManager.updateAfterTaskFinished(job,
job.getTask("nativeTask-" + i),
new TaskResultImpl(null, "OK", null, 0));
}
System.out.println("Remove job");
long start = System.currentTimeMillis();
dbManager.removeJob(job.getId(), 0, true);
System.out.println("Remove time (tasks: " + tasksNumber + ")" + (System.currentTimeMillis() - start));
checkAllEntitiesDeleted();
}
private void setAttributes(Task task) throws Exception {
task.addGenericInformation("k1", "v1");
task.addGenericInformation("k2", "v2");
SimpleScript script = new SimpleScript(task.getName() + "selection script",
"javascript",
new String[] { "param1", "param2" });
SelectionScript ss1 = new SelectionScript(script, true);
SelectionScript ss2 = new SelectionScript(script, false);
task.addSelectionScript(ss1);
task.addSelectionScript(ss2);
task.setPreScript(new SimpleScript(task.getName() + "pre script",
"javascript",
new String[] { "param1", "param2" }));
task.setPostScript(new SimpleScript(task.getName() + "post script",
"javascript",
new String[] { "param1", "param2" }));
task.setCleaningScript(new SimpleScript(task.getName() + "clean script",
"javascript",
new String[] { "param1", "param2" }));
task.setFlowScript(FlowScript.createContinueFlowScript());
task.addInputFiles("f1", InputAccessMode.TransferFromGlobalSpace);
task.addInputFiles("f2", InputAccessMode.TransferFromInputSpace);
task.addInputFiles("f3", InputAccessMode.TransferFromUserSpace);
task.addOutputFiles("f1", OutputAccessMode.TransferToGlobalSpace);
task.addOutputFiles("f2", OutputAccessMode.TransferToOutputSpace);
task.addOutputFiles("f3", OutputAccessMode.TransferToUserSpace);
}
private void checkAllEntitiesDeleted(String... skipClasses) {
Set<String> skip = ImmutableSet.copyOf(skipClasses);
Session session = dbManager.getSessionFactory().openSession();
try {
for (ClassMetadata metadata : session.getSessionFactory().getAllClassMetadata().values()) {
if (!skip.contains(metadata.getEntityName())) {
System.out.println("Check " + metadata.getEntityName());
List<Object> list = session.createCriteria(metadata.getEntityName()).list();
Assert.assertEquals("Unexpected " + metadata.getEntityName(), 0, list.size());
}
}
} finally {
session.close();
}
}
}