/*
* Copyright 2015-2016 OpenCB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opencb.opencga.analysis;
import org.apache.tools.ant.types.Commandline;
import org.opencb.commons.datastore.core.QueryResult;
import org.opencb.commons.utils.StringUtils;
import org.opencb.opencga.catalog.models.tool.Execution;
import org.opencb.opencga.catalog.monitor.exceptions.ExecutionException;
import org.opencb.opencga.catalog.monitor.executors.old.ExecutorManager;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.models.File;
import org.opencb.opencga.catalog.models.Job;
import org.opencb.opencga.core.common.TimeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.util.*;
import java.util.stream.Collectors;
/**
* Created on 30/11/15
*
* @author Jacobo Coll <jacobo167@gmail.com>
*/
public class JobFactory {
private final CatalogManager catalogManager;
protected static Logger logger = LoggerFactory.getLogger(JobFactory.class);
public JobFactory(CatalogManager catalogManager) {
this.catalogManager = catalogManager;
}
public QueryResult<Job> createJob(ToolManager toolManager, Map<String, List<String>> params, long studyId, String jobName, String description,
File outDir, List<Long> inputFiles, String sessionId)
throws AnalysisExecutionException, CatalogException {
return createJob(toolManager, params, studyId, jobName, description, outDir, inputFiles, sessionId, false);
}
public QueryResult<Job> createAndExecuteJob(ToolManager toolManager, Map<String, List<String>> params, long studyId, String jobName, String description,
File outDir, List<Long> inputFiles, String sessionId)
throws AnalysisExecutionException, CatalogException {
return createJob(toolManager, params, studyId, jobName, description, outDir, inputFiles, sessionId, true);
}
/**
* Create a catalog Job given a {@link ToolManager} and a set of params.
*
* Requires a ToolManager to create the command line.
*
* @param toolManager {@link ToolManager} of the tool
* @param params Params to use with the tool
* @param studyId StudyId where to create the job
* @param jobName Job name
* @param description Job description
* @param outDir Output directory
* @param inputFiles Input files
* @param sessionId User sessionId
* @param execute Execute job locally before create
* @return New catalog job
* @throws AnalysisExecutionException
* @throws CatalogException
*/
public QueryResult<Job> createJob(ToolManager toolManager, Map<String, List<String>> params, long studyId, String jobName, String description,
File outDir, List<Long> inputFiles, String sessionId, boolean execute)
throws AnalysisExecutionException, CatalogException {
Execution execution = toolManager.getExecution();
String executable = execution.getExecutable();
// Create temporal Outdir
String randomString = "J_" + StringUtils.randomString(10);
URI temporalOutDirUri = catalogManager.createJobOutDir(studyId, randomString, sessionId);
params.put(execution.getOutputParam(), Arrays.asList(temporalOutDirUri.getPath()));
// Create commandLine
String commandLine = toolManager.createCommandLine(executable, params);
logger.debug("Command line : {}", commandLine);
Map<String, String> plainParams = params.entrySet().stream().collect(Collectors.toMap(
Map.Entry::getKey,
entry -> entry.getValue().stream().collect(Collectors.joining(",")))
);
HashMap<String, Object> attributes = new HashMap<>();
attributes.put("plugin", toolManager.isPlugin()); //TODO: Save type of tool in a better way
return createJob(studyId, jobName, toolManager.getAnalysisName(), execution.getId(), plainParams, commandLine, description, outDir,
temporalOutDirUri, inputFiles, randomString, attributes, new HashMap<>(), sessionId,
false, execute);
}
@Deprecated
public QueryResult<Job> createJob(long studyId, String jobName, String toolName, String description,
File outDir, List<Long> inputFiles, final String sessionId,
String randomString, URI temporalOutDirUri, String commandLine,
boolean execute, boolean simulate, Map<String, Object> attributes,
Map<String, Object> resourceManagerAttributes)
throws AnalysisExecutionException, CatalogException {
Map<String, String> params = getParamsFromCommandLine(commandLine);
return createJob(studyId, jobName, toolName, "", params, commandLine, description, outDir, temporalOutDirUri, inputFiles,
randomString, attributes, resourceManagerAttributes, sessionId, simulate, execute);
}
/**
* Create a catalog Job given a commandLine and the rest of parameters.
*
* @param studyId Study id
* @param jobName Job name
* @param toolName Tool name
* @param executor Tool executor name
* @param params Map of params
* @param commandLine Command line to execute
* @param description Job description (optional)
* @param outDir Final output directory
* @param temporalOutDirUri Temporal output directory
* @param inputFiles List of input files
* @param jobSchedulerName Name of the job in the job scheduler. Usually a random string.
* @param attributes Optional attributes
* @param resourceManagerAttributes Optional resource manager attributes
* @param sessionId User sessionId
* @param simulate Simulate job creation. Do not create any job in catalog.
* @param execute Execute job locally before create
* @return New catalog job
* @throws AnalysisExecutionException
* @throws CatalogException
*/
public QueryResult<Job> createJob(long studyId, String jobName, String toolName, String executor, Map<String, String> params, String commandLine, String description,
File outDir, URI temporalOutDirUri, List<Long> inputFiles, String jobSchedulerName, Map<String, Object> attributes, Map<String, Object> resourceManagerAttributes, final String sessionId,
boolean simulate, boolean execute)
throws AnalysisExecutionException, CatalogException {
logger.debug("Creating job {}: simulate {}, execute {}", jobName, simulate, execute);
long start = System.currentTimeMillis();
QueryResult<Job> jobQueryResult;
if (resourceManagerAttributes == null) {
resourceManagerAttributes = new HashMap<>();
}
resourceManagerAttributes.put(Job.JOB_SCHEDULER_NAME, jobSchedulerName);
if (simulate) { //Simulate a job. Do not create it.
jobQueryResult = new QueryResult<>("simulatedJob", (int) (System.currentTimeMillis() - start), 1, 1, "", "", Collections.singletonList(
new Job(-10, jobName, catalogManager.getUserIdBySessionId(sessionId), toolName,
TimeUtils.getTime(), description, start, System.currentTimeMillis(), "", commandLine, -1,
new Job.JobStatus(Job.JobStatus.PREPARED), -1, outDir.getId(), inputFiles, Collections.emptyList(),
null, attributes, resourceManagerAttributes)));
} else {
if (execute) {
/** Create a RUNNING job in CatalogManager **/
jobQueryResult = catalogManager.createJob(studyId, jobName, toolName, description, executor, params, commandLine, temporalOutDirUri,
outDir.getId(), inputFiles, null, attributes, resourceManagerAttributes, new Job.JobStatus(Job.JobStatus.RUNNING),
System.currentTimeMillis(), 0, null, sessionId);
Job job = jobQueryResult.first();
//Execute job in local
// LocalExecutorManager executorManager = new LocalExecutorManager(catalogManager, sessionId);
// jobQueryResult = executorManager.run(job);
try {
ExecutorManager.execute(catalogManager, job, sessionId, "LOCAL");
} catch (ExecutionException e) {
throw new AnalysisExecutionException(e.getCause());
} catch (IOException e) {
throw new AnalysisExecutionException(e.getCause());
}
jobQueryResult = catalogManager.getJob(job.getId(), null, sessionId);
} else {
/** Create a PREPARED job in CatalogManager **/
jobQueryResult = catalogManager.createJob(studyId, jobName, toolName, description, executor, params, commandLine, temporalOutDirUri,
outDir.getId(), inputFiles, null, attributes, resourceManagerAttributes, new Job.JobStatus(Job.JobStatus.PREPARED), 0, 0, null, sessionId);
}
}
return jobQueryResult;
}
public static Map<String, String> getParamsFromCommandLine(String commandLine) {
String[] args = Commandline.translateCommandline(commandLine);
Map<String, String> params = new HashMap<>();
for (int i = 0; i < args.length; i++) {
if (args[i].startsWith("-")) {
String key = args[i].replaceAll("^--?", "");
String value;
if (args.length == i + 1 || args[i + 1].startsWith("-")) {
value = "";
} else {
value = args[i + 1];
i++;
}
params.put(key, value);
}
}
return params;
}
}