/**
* @author briandoconnor@gmail.com
*
* The WorkflowLauncher is responsible for launching workflows with or without metadata writeback.
*
*/
package net.sourceforge.seqware.pipeline.plugins;
import io.seqware.pipeline.SqwKeys;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import joptsimple.ArgumentAcceptingOptionSpec;
import net.sourceforge.seqware.common.metadata.MetadataDB;
import net.sourceforge.seqware.common.module.ReturnValue;
import net.sourceforge.seqware.common.module.ReturnValue.ExitStatus;
import net.sourceforge.seqware.common.util.Log;
import net.sourceforge.seqware.common.util.TabExpansionUtil;
import net.sourceforge.seqware.common.util.workflowtools.WorkflowInfo;
import net.sourceforge.seqware.pipeline.bundle.Bundle;
import net.sourceforge.seqware.pipeline.bundle.BundleInfo;
import net.sourceforge.seqware.pipeline.plugin.Plugin;
import net.sourceforge.seqware.pipeline.plugin.PluginInterface;
import org.apache.commons.io.FileUtils;
import org.openide.util.lookup.ServiceProvider;
/**
* <p>
* BundleManager class.
* </p>
*
* @author boconnor
*
* ProviderFor(PluginInterface.class)
*
* TODO: improve the use of workflow-accession, see https://jira.oicr.on.ca/browse/SEQWARE-539 TODO: the param listing won't
* currently work for a local provisioned directory
* @version $Id: $Id
*/
@ServiceProvider(service = PluginInterface.class)
public class BundleManager extends Plugin {
ReturnValue ret = new ReturnValue();
private final ArgumentAcceptingOptionSpec<String> outFile;
/**
* <p>
* Constructor for BundleManager.
* </p>
*/
public BundleManager() {
super();
parser.acceptsAll(Arrays.asList("help", "h", "?"), "Optional: Provides this help message.");
parser.acceptsAll(Arrays.asList("bundle", "b"),
"The path to a bundle zip file, can specify this or the workflow-run-accession of an already-installed bundle.")
.withRequiredArg();
parser.acceptsAll(Arrays.asList("list", "l"), "Optional: List the workflows contained in this bundle.");
parser.acceptsAll(
Arrays.asList("list-installed", "list-install"),
"Optional: List the workflows contained in this bundle. The database/webservice must be enabled in your .seqware/settings for this option to work.");
parser.acceptsAll(
Arrays.asList("workflow-accession", "wa"),
"Optional: The sw_accession of the workflow. Specify this or the workflow, version, and bundle. Currently used in conjunction with the list-workflow-params for now.")
.withRequiredArg();
parser.acceptsAll(
Arrays.asList("list-workflow-params", "list-params"),
"Optional: List the parameters for a given workflow and version. You need to supply a workflow accession and you need a database/webservice enabled in your .seqware/settings for this option to work.");
parser.acceptsAll(Arrays.asList("validate", "v"), "Optional: Run a light basic validation on this bundle.");
parser.acceptsAll(
Arrays.asList("install", "i"),
"Optional: if the --bundle param points to a .zip file then the install process will first unzip into the directory specified by the directory defined by "
+ SqwKeys.SW_BUNDLE_DIR.getSettingKey()
+ " in the .seqware/settings file (skipping files that already exit). It will then copy the whole zip file to the "
+ SqwKeys.SW_BUNDLE_REPO_DIR.getSettingKey()
+ " which can be a directory or S3 prefix (the copy will be skipped if the file is already at this location). It will finish this process by installing this bundle in the database with the permanent_bundle_location pointed to the zip file location and current_working_dir pointed to the unzipped location. If the --bundle param point to a directory then this will first create a zip of the bundle and place it in "
+ SqwKeys.SW_BUNDLE_REPO_DIR.getSettingKey()
+ ". It will then install this bundle in the database with the permanent_bundle_location pointed to the zip file location and current_working_dir pointed to the unzipped location. The method (direct database or web service) and server location of the SeqWare MetaDB is controlled via the .seqware/settings file.");
parser.acceptsAll(
Arrays.asList("install-zip-only", "izo"),
"Optional: This will suppress the unzipping of a zip file, it is only valid if the --bundle points to a zip file and not a directory. It will take a workflow bundle zip file, copy it to the "
+ SqwKeys.SW_BUNDLE_REPO_DIR.getSettingKey()
+ " location, and then installs that workflow into the database. Only the permanent_bundle_location location will be defined, the current_working_dir will be null. (PROBLEM: can't read the metadata.xml if the workflow zip isn't unzipped!)");
parser.acceptsAll(
Arrays.asList("install-dir-only", "ido"),
"Optional: This will suppress the creation of a zip file from a workflow bundle directory. It will simply install the workflow into the database and set the current_working_dir but leave permanent_bundle_location null.");
parser.acceptsAll(
Arrays.asList("path-to-package", "p"),
"Optional: When combined with a bundle zip file specified via --bundle this option specifies an input directory to zip to create a bundle output file.")
.withRequiredArg();
parser.acceptsAll(
Arrays.asList("workflow", "w"),
"The name of the workflow to be used. This must be used in conjunction with a version and bundle. Will restrict action to this workflow and version if specified.")
.withRequiredArg();
parser.acceptsAll(
Arrays.asList("version", "workflow-version"),
"The workflow version to be used. This must be used in conjunction with a version and bundle. Will restrict action to this workflow and version if specified.")
.withRequiredArg();
parser.acceptsAll(Arrays.asList("download"),
"Downloads a workflow bundle zip. This must be used in conjunction with a workflow name and version.");
parser.acceptsAll(Arrays.asList("download-url"), "Downloads a workflow bundle zip from a URL to the local directory.")
.withRequiredArg();
parser.acceptsAll(Arrays.asList("metadata", "m"), "Specify the path to the metadata.xml file.").withRequiredArg();
parser.acceptsAll(Arrays.asList("human-expanded", "he"), "Optional: will print output in expanded human friendly format");
parser.acceptsAll(Arrays.asList("human-aligned", "ha"), "Optional: will print output in aligned human friendly format");
this.outFile = parser.acceptsAll(Arrays.asList("out"), "Optional: Will output a list of workflows installed by sw_accession")
.withRequiredArg();
ret.setExitStatus(ReturnValue.SUCCESS);
}
/*
* (non-Javadoc)
*
* @see net.sourceforge.seqware.pipeline.plugin.PluginInterface#init()
*/
/**
* {@inheritDoc}
*
* @return
*/
@Override
public ReturnValue init() {
return ret;
}
/*
* (non-Javadoc)
*
* @see net.sourceforge.seqware.pipeline.plugin.PluginInterface#do_test()
*/
/**
* {@inheritDoc}
*
* @return
*/
@Override
public ReturnValue do_test() {
// TODO Auto-generated method stub
return ret;
}
/*
* (non-Javadoc)
*
* @see net.sourceforge.seqware.pipeline.plugin.PluginInterface#do_run()
*/
/**
* {@inheritDoc}
*
* @return
*/
@Override
public ReturnValue do_run() {
if (options.has("install") && options.has("install-dir-only")) {
println("Combination of both install and install-dir-only not recognized");
ret.setExitStatus(ReturnValue.INVALIDPARAMETERS);
return ret;
}
// setup bundle object
Bundle b = new Bundle(metadata, config);
// metadata file to use if provided
String specificMetadataStr = (String) options.valueOf("metadata");
File specificMetadataFile = null;
if (specificMetadataStr != null) {
specificMetadataFile = new File(specificMetadataStr);
}
List<String> workflowsInstalled = new ArrayList<>();
// list the contents of the bundle
if (options.has("bundle") && (options.has("list") || options.has("help"))) {
String bundlePath = (String) options.valueOf("bundle");
File bundle = new File(bundlePath);
BundleInfo bi = b.getBundleInfo(bundle, specificMetadataFile);
println("\nList Workflows:\n");
for (WorkflowInfo wi : bi.getWorkflowInfo()) {
println(" Workflow:");
println(" Name: " + wi.getName());
println(" Version: " + wi.getVersion());
println(" Description: " + wi.getDescription());
if (wi.getTemplatePath() != null) println(" Template Path: " + wi.getTemplatePath());
if (wi.getWorkflowClass() != null) println(" Workflow Class: " + wi.getWorkflowClass());
println(" Config Path: " + wi.getConfigPath());
println(" Requirements Compute: " + wi.getComputeReq() + " Memory: " + wi.getMemReq() + " Network: " + wi.getNetworkReq()
+ "\n");
}
} else if (options.has("bundle") && options.has("validate")) {
println("Validating Bundle");
String bundlePath = (String) options.valueOf("bundle");
File bundle = new File(bundlePath);
ret = b.validateBundle(bundle);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Bundle Validates!");
}
} else if (options.has("bundle") && options.has("path-to-package")) {
println("Validating Bundle structure");
File bundleDir = new File((String) options.valueOf("path-to-package"));
File bundleZip = new File((String) options.valueOf("bundle"));
ret = b.validateBundle(bundleDir);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Packaging Bundle");
ret = b.packageBundle(bundleDir, bundleZip);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Bundle has been packaged to " + bundleZip.getAbsolutePath());
}
}
} else if (options.has("bundle") && options.has("install")) {
if (killIfDirectDB()) {
return ret;
}
println("Installing Bundle");
String bundleFile = (String) options.valueOf("bundle");
println("Bundle: " + bundleFile);
ret = b.installBundle(new File(bundleFile), specificMetadataFile, workflowsInstalled);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Bundle Has Been Installed to the MetaDB and Provisioned to " + options.valueOf("bundle") + "!");
}
} else if (options.has("bundle") && options.has("install-zip-only")) {
if (killIfDirectDB()) {
return ret;
}
println("Installing Bundle (Zip Bundle Archive Only)");
String bundleFile = (String) options.valueOf("bundle");
println("Bundle: " + bundleFile);
if (bundleFile.endsWith(".zip")) {
ret = b.installBundleZipOnly(new File(bundleFile), specificMetadataFile, workflowsInstalled);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Bundle Has Been Installed to the MetaDB and Provisioned to " + options.valueOf("bundle") + "!");
}
} else {
Log.error("Attempting to install a workflow bundle zip file but the bundle does not end in .zip! " + bundleFile);
ret.setExitStatus(ReturnValue.FAILURE);
}
} else if (options.has("bundle") && options.has("install-dir-only")) {
if (killIfDirectDB()) {
return ret;
}
println("Installing Bundle (Working Directory Only)");
String bundleFile = (String) options.valueOf("bundle");
println("Bundle: " + bundleFile);
File bundleDir = new File(bundleFile);
if (bundleDir.exists() && bundleDir.isDirectory()) {
ret = b.installBundleDirOnly(bundleDir, specificMetadataFile, workflowsInstalled);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Bundle Has Been Installed to the MetaDB and Provisioned to " + options.valueOf("bundle") + "!");
}
} else {
Log.error("Attempting to install a workflow bundle from an unzipped bundle directory but the bundle does not exit or point to a directory! "
+ bundleFile);
ret.setExitStatus(ReturnValue.FAILURE);
}
} else if (options.has("list-installed")) {
String localParams = metadata.listInstalledWorkflows();
final String nameVersionCreationDateSeqWareAccession = "Name\tVersion\tCreation Date\tSeqWare Accession\tDescription\tCurrent Working Directory\tBundle Location";
if (options.has("human-expanded")) {
localParams = nameVersionCreationDateSeqWareAccession + "\n" + localParams;
localParams = TabExpansionUtil.expansion(localParams);
if (localParams.trim().isEmpty()) {
println("No workflows installed.");
} else {
println(localParams);
}
} else if (options.has("human-aligned")) {
localParams = nameVersionCreationDateSeqWareAccession + "\n" + localParams;
localParams = TabExpansionUtil.aligned(localParams);
println(localParams);
} else {
println("=====================================================");
println("===============INSTALLED WORKFLOWS===================");
println("=====================================================");
println(nameVersionCreationDateSeqWareAccession);
println("-----------------------------------------------------");
println(localParams);
println("-----------------------------------------------------");
}
// ((options.has("workflow") && options.has("version") && options.has("bundle")) || options.has("workflow-accession"))
} else if (options.has("list-workflow-params") && options.has("workflow-accession")) {
String localParams = metadata.listInstalledWorkflowParams((String) options.valueOf("workflow-accession"));
println(localParams);
} else if (options.has("workflow") && options.has("version") && options.has("download")) {
println("Downloading Bundle");
String workflowName = (String) options.valueOf("workflow");
println("Workflow: " + workflowName);
String version = (String) options.valueOf("version");
println("Version: " + version);
int accession = metadata.getWorkflowAccession(workflowName, version);
Map<String, String> info = metadata.get_workflow_info(accession);
String sourceFile = info.get("permanent_bundle_location");
String targetDir = new File(".").getAbsolutePath();
ret = b.copyBundle(sourceFile, targetDir);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Bundle has been copied to the current directory");
}
} else if (options.has("download-url")) {
String sourceFile = (String) options.valueOf("download-url");
String targetDir = new File(".").getAbsolutePath();
ret = b.copyBundle(sourceFile, targetDir);
if (ret.getExitStatus() == ReturnValue.SUCCESS) {
println("Bundle has been copied to the current directory");
}
} else {
println("Combination of parameters not recognized!");
println(this.get_syntax());
ret.setExitStatus(ReturnValue.INVALIDPARAMETERS);
}
if (workflowsInstalled.size() > 0 && options.has(this.outFile)) {
try {
File file = new File(options.valueOf(this.outFile));
FileUtils.writeLines(file, workflowsInstalled);
} catch (IOException ex) {
return new ReturnValue(ExitStatus.FILENOTWRITABLE);
}
}
return ret;
}
/*
* (non-Javadoc)
*
* @see net.sourceforge.seqware.pipeline.plugin.PluginInterface#clean_up()
*/
/**
* {@inheritDoc}
*
* @return
*/
@Override
public ReturnValue clean_up() {
// TODO Auto-generated method stub
return ret;
}
/**
* <p>
* get_description.
* </p>
*
* @return a {@link java.lang.String} object.
*/
@Override
public String get_description() {
return ("A plugin that lets you create, test, and install workflow bundles.");
}
private boolean killIfDirectDB() {
if (this.metadata instanceof MetadataDB) {
Log.stdout("Bundle installation is not supported using a database connection to the MetaDB");
ret = new ReturnValue(ReturnValue.RUNTIMEEXCEPTION);
return true;
}
return false;
}
}