/**
* Copyright (c) 2015 Genome Research Ltd.
*
* Author: Cancer Genome Project cgpit@sanger.ac.uk
*
* This file is part of WwDocker.
*
* WwDocker is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation; either version 3 of the License, or (at your option) any
* later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* 1. The usage of a range of years within a copyright statement contained within
* this distribution should be interpreted as being equivalent to a list of years
* including the first and last year specified and all consecutive years between
* them. For example, a copyright statement that reads 'Copyright (c) 2005, 2007-
* 2009, 2011-2012' should be interpreted as being identical to a statement that
* reads 'Copyright (c) 2005, 2007, 2008, 2009, 2011, 2012' and a copyright
* statement that reads "Copyright (c) 2005-2012' should be interpreted as being
* identical to a statement that reads 'Copyright (c) 2005, 2006, 2007, 2008,
* 2009, 2010, 2011, 2012'."
*/
package uk.ac.sanger.cgp.wwdocker.workflow;
import com.jcraft.jsch.Session;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import uk.ac.sanger.cgp.wwdocker.Config;
import uk.ac.sanger.cgp.wwdocker.actions.Local;
import static uk.ac.sanger.cgp.wwdocker.actions.Local.execCommand;
import uk.ac.sanger.cgp.wwdocker.actions.Remote;
import uk.ac.sanger.cgp.wwdocker.actions.Utils;
import uk.ac.sanger.cgp.wwdocker.interfaces.Workflow;
/**
*
* @author kr2
*/
public class DEWorkflow implements Workflow {
private static final Logger logger = LogManager.getLogger();
BaseConfiguration config;
private static final String[] logSearchCmd = {"find oozie-*/ -type f | grep 'gtdownload.*log$'",
"find oozie-*/shared_workspace/oozie-*/generated-scripts -type f",
"find oozie-*/shared_workspace/oozie-*/delly_results -type f | grep '.log$'",};
public DEWorkflow(BaseConfiguration config) {
this.config = config;
}
@Override
public String[] getFindLogsCmds() {
return logSearchCmd;
}
@Override
public List filesToPush(File iniFile) {
List files = new ArrayList();
// see example in SangerWorkflow if we need this later
return files;
}
@Override
public List filesToPull(File iniFile) {
List files = new ArrayList();
//@TODO
return files;
}
@Override
public String baseDockerCommand(BaseConfiguration config, String extras) {
//File workflow = Paths.get(config.getString("workflowDir"), config.getString("workflow").replaceAll(".*/", "").replaceAll("\\.zip$", "")).toFile();
// probably want to clean the data store before we write the ini file
//docker run --rm -h master -v /cgp/datastore:/datastore -v /cgp/workflows/Workflow_Bundle_SangerPancancerCgpCnIndelSnvStr_1.0.5.1_SeqWare_1.1.0-alpha.5:/workflow -i seqware/seqware_whitestar_pancancer rm -rf /datastore/
String command = "docker run --rm -h master";
// this is an oddity of the DKFZ version to ensure that paths are handled correctly in the inner docker call
command = command.concat(" -v ").concat(config.getString("datastoreDir")).concat(":").concat(config.getString("datastoreDir"));
command = command.concat(" -v ").concat(config.getString("workflowDir")).concat(":/workflow");
if(extras != null) {
command = command.concat(extras);
}
command = command.concat(" ").concat(seqwareWhiteStarImage(config));
return command;
}
@Override
public int runDocker(BaseConfiguration config, File iniFile) {
/*
* docker run --rm -h master -t
* -v /cgp/datastore:/datastore
* -v /cgp/Workflow_Bundle_SangerPancancerCgpCnIndelSnvStr_1.0.5.1_SeqWare_1.1.0-alpha.5:/workflow
* -i seqware/seqware_whitestar_pancancer
* seqware bundle launch
* --no-metadata
* --engine whitestar-parallel
* --dir /workflow
* --ini /datastore/testRun.ini
*/
String extras = " -v /var/run/docker.sock:/var/run/docker.sock";
extras = extras.concat(" -v ").concat(config.getString("datastoreDir")).concat("/").concat(iniFile.getName()).concat(":/workflow.ini");
// don't add the pem key here, just standardise in the ini file template
String command = baseDockerCommand(config, extras);
command = command.concat(" bash -c \"sed -i 's|/datastore|" + config.getString("datastoreDir") + "|g' /home/seqware/.seqware/settings ;");
command = command.concat(" sed -i 's|OOZIE_RETRY_MAX=.*|OOZIE_RETRY_MAX=0|' /home/seqware/.seqware/settings ;");
command = command.concat(" seqware bundle launch --no-metadata --engine whitestar");
command = command.concat(" --dir /workflow/").concat(config.getString("workflow").replaceAll(".*/", "").replaceAll("\\.zip$", ""));
command = command.concat(" --ini /workflow.ini");
command = command.concat("\""); // close quotes on bash command
// this may need to be more itelligent than just the exit code
return execCommand(command, Config.getEnvs(config), true);
}
@Override
public boolean provisionHost(String host, BaseConfiguration config, File thisJar, File tmpConf, String mode, Map<String, String> envs) throws InterruptedException {
boolean provisioned = false;
String remoteWorkflowDir = config.getString("workflowDir");
String localSeqwareJar = config.getString("seqware");
String localWorkflowZip = config.getString("workflow");
File jreDist = Utils.expandUserFile(config, "jreDist", true);
File remoteSeqwareJar = new File(remoteWorkflowDir.concat("/").concat(localSeqwareJar.replaceAll(".*/", "")));
File remoteWorkflowZip = new File(remoteWorkflowDir.concat("/").concat(localWorkflowZip.replaceAll(".*/", "")));
String[] pullDockerImages = config.getStringArray("pullDockerImages");
String[] curlDockerImages = config.getStringArray("curlDockerImages");
String[] pushDockerImages = config.getStringArray("pushDockerImages");
String optDir = "/opt/wwdocker";
String workerLog = config.getString("log4-worker");
File localTmp = Utils.expandUserDirPath(config, "primaryLargeTmp", true);
List<String> createPaths = new ArrayList();
createPaths.add("/opt/wwdocker");
createPaths.add("/opt/wwdocker/jre");
createPaths.add(remoteWorkflowDir);
createPaths.add(config.getString("datastoreDir"));
Session ssh = Remote.getSession(config, host);
Remote.createPaths(ssh, createPaths);
Remote.chmodPaths(ssh, "a+wrx", createPaths, true);
Remote.cleanFiles(ssh, new String[]{config.getString("log4-delete")});
Remote.cleanupOldImages(ssh); // incase lots of stale ones are already present
if(Local.pushFileSetToHost(pushDockerImages, host, "/opt/wwdocker", envs, ssh, null) != 0) {
return provisioned;
}
String[] pushedDockerImages = new String[pushDockerImages.length];
for(int i=0; i<pushDockerImages.length; i++) {
pushedDockerImages[i] = Paths.get("/opt/wwdocker", new File(pushDockerImages[i]).getName()).toFile().getPath();
}
if (Remote.dockerPull(ssh, pullDockerImages) != 0
|| Remote.dockerLoad(ssh, curlDockerImages, remoteWorkflowDir) != 0
|| Remote.dockerLoad(ssh, pushedDockerImages, remoteWorkflowDir) != 0) {
return provisioned;
}
Remote.cleanupOldImages(ssh); // incase lots of stale ones are already present
if(Remote.curl(ssh, localSeqwareJar, remoteWorkflowDir) == null) {
return provisioned;
}
Local.pushToHost(jreDist.getAbsolutePath(), host, optDir, envs, ssh, localTmp);
if (Remote.expandJre(ssh, jreDist) != 0) {
return provisioned;
}
Remote.curl(ssh, localWorkflowZip, remoteWorkflowDir);
if (Remote.expandWorkflow(ssh, remoteWorkflowZip, remoteSeqwareJar, remoteWorkflowDir) != 0) {
return provisioned;
}
//String workflowBase = remoteWorkflowZip.getName().replaceAll("\\.zip$", "");
//Path gnosDest = Paths.get(remoteWorkflowDir, workflowBase);
if (Local.pushToHost(thisJar.getAbsolutePath(), host, optDir, envs, ssh, localTmp) != 0 // this jar file
|| Local.pushToHost(workerLog, host, optDir, envs, ssh, localTmp) != 0 // worker log config
|| Local.pushToHost(tmpConf.getAbsolutePath(), host, optDir, envs, ssh, localTmp) != 0 // config file
|| Remote.chmodPath(ssh, "go-wrx", optDir.concat("/*"), true) != 0 // file will have passwords
|| Local.pushFileSetToHost(Utils.getGnosKeys(config), host, config.getString("datastoreDir"), envs, ssh, localTmp) != 0 // GNOS keys, note DATASTORE
|| Remote.chmodPath(ssh, "a+r", config.getString("datastoreDir").concat("/*.pem"), false) != 0 // need to ensure these are readable within the image
|| Remote.startWorkerDaemon(ssh, thisJar.getName(), tmpConf.getName(), mode) != 0) {
return provisioned;
}
provisioned = true;
return provisioned;
}
@Override
public int cleanDockerPath(BaseConfiguration config) {
String command = baseDockerCommand(config, null);
String datastore = config.getString("datastoreDir");
List<String> args = new ArrayList(Arrays.asList(command.split(" ")));
args.add("/bin/bash");
args.add("-c");
args.add("rm -rf "
+datastore+ "/oozie-* "
+datastore+ "/*.ini "
+datastore+ "/logs.tar.gz "
+datastore+ "/toInclude.lst"
+datastore+ "/DEWorkflowData/dkfz/gtdownload-*.log"
);
ProcessBuilder pb = new ProcessBuilder(args);
Map<String, String> pEnv = pb.environment();
pEnv.putAll(Config.getEnvs(config));
logger.info("Executing: " + String.join(" ", args));
int exitCode = -1;
Process p = null;
try {
p = pb.start();
String progErr = IOUtils.toString(p.getErrorStream());
String progOut = IOUtils.toString(p.getInputStream());
exitCode = p.waitFor();
Utils.logOutput(progErr, Level.ERROR);
Utils.logOutput(progOut, Level.TRACE);
} catch(InterruptedException | IOException e) {
logger.error(e.getMessage(), e);
}
finally {
if(p != null) {
p.destroy();
exitCode = p.exitValue();
}
}
return exitCode;
}
}