/**
* Copyright 2007-2008 University Of Southern California
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.isi.pegasus.planner.refiner;
import edu.isi.pegasus.common.logging.LoggingKeys;
import java.util.Iterator;
import java.util.Set;
import edu.isi.pegasus.planner.classes.ADag;
import edu.isi.pegasus.planner.classes.PegasusBag;
import edu.isi.pegasus.planner.classes.PlannerOptions;
import edu.isi.pegasus.common.logging.LogManager;
import edu.isi.pegasus.common.util.FileUtils;
import edu.isi.pegasus.planner.catalog.TransformationCatalog;
import edu.isi.pegasus.planner.catalog.site.classes.SiteStore;
import edu.isi.pegasus.planner.classes.NameValue;
import edu.isi.pegasus.planner.classes.PlannerCache;
import java.io.File;
import java.io.IOException;
import java.util.LinkedHashSet;
/**
* The central class that calls out to the various other components of Pegasus.
*
* @author Karan Vahi
* @author Gaurang Mehta
* @version $Revision$
*/
public class MainEngine
extends Engine {
/**
* The basename of the directory that contains the submit files for the
* cleanup DAG that for the concrete dag generated for the workflow.
*/
public static final String CLEANUP_DIR = "cleanup";
public static String CATALOGS_DIR_BASENAME = "catalogs";
/**
* The Original Dag object which is constructed by parsing the dag file.
*/
private ADag mOriginalDag;
/**
* The reduced Dag object which is got from the Reduction Engine.
*/
private ADag mReducedDag;
/**
* The cleanup dag for the final concrete dag.
*/
private ADag mCleanupDag;
/**
* The pools on which the Dag should be executed as specified by the user.
*/
private Set mExecPools;
/**
* The bridge to the Replica Catalog.
*/
private ReplicaCatalogBridge mRCBridge;
/**
* The handle to the InterPool Engine that calls out to the Site Selector
* and maps the jobs.
*/
private InterPoolEngine mIPEng;
/**
* The handle to the Reduction Engine that performs reduction on the graph.
*/
private DataReuseEngine mRedEng;
/**
* The handle to the Transfer Engine that adds the transfer nodes in the
* graph to transfer the files from one site to another.
*/
private TransferEngine mTransEng;
/**
* The engine that ends up creating random directories in the remote
* execution pools.
*/
private CreateDirectory mCreateEng;
/**
* The engine that ends up creating the cleanup dag for the dag.
*/
private RemoveDirectory mRemoveEng;
/**
* The handle to the node collapser.
*/
private NodeCollapser mNodeCollapser;
/**
* This constructor initialises the class variables to the variables
* passed. The pool names specified should be present in the pool.config file
*
* @param orgDag the dag to be worked on.
* @param bag the bag of initialization objects
*/
public MainEngine( ADag orgDag, PegasusBag bag ) {
super( bag );
mOriginalDag = orgDag;
mExecPools = (Set)mPOptions.getExecutionSites();
mOutputPool = mPOptions.getOutputSite();
if (mOutputPool != null && mOutputPool.length() > 0) {
Engine.mOutputPool = mOutputPool;
}
}
/**
* The main function which calls the other engines and does the necessary work.
*
* @return the planned worflow.
*/
public ADag runPlanner() {
String abstractWFName = mOriginalDag.getAbstractWorkflowName();
//create the main event refinement event
mLogger.logEventStart( LoggingKeys.EVENT_PEGASUS_REFINEMENT,
LoggingKeys.DAX_ID,
abstractWFName );
//refinement process starting
mOriginalDag.setWorkflowRefinementStarted( true );
String message = null;
mRCBridge = new ReplicaCatalogBridge( mOriginalDag, mBag );
//PM-1047 copy all catalog file sources to submit directory
copyCatalogFiles( mBag.getHandleToSiteStore(),
mBag.getHandleToTransformationCatalog(),
mRCBridge,
new File( this.mPOptions.getSubmitDirectory(), CATALOGS_DIR_BASENAME ));
//lock down on the workflow task metrics
//the refinement process will not update them
mOriginalDag.getWorkflowMetrics().lockTaskMetrics( true );
//check for cyclic dependencies
mLogger.logEventStart( LoggingKeys.EVENT_PEGASUS_CYCLIC_DEPENDENCY_CHECK, LoggingKeys.DAX_ID, abstractWFName );
if( mOriginalDag.hasCycles() ){
NameValue nv = mOriginalDag.getCyclicEdge();
throw new RuntimeException( "Cyclic dependency detected " + nv.getKey() + " -> " + nv.getValue() );
}
mLogger.logEventCompletion();
mRedEng = new DataReuseEngine( mOriginalDag, mBag );
mReducedDag = mRedEng.reduceWorkflow(mOriginalDag, mRCBridge );
//unmark arg strings
//unmarkArgs();
mOriginalDag = null;
mLogger.logEventStart( LoggingKeys.EVENT_PEGASUS_SITESELECTION, LoggingKeys.DAX_ID, abstractWFName );
mIPEng = new InterPoolEngine( mReducedDag, mBag );
mIPEng.determineSites();
mBag = mIPEng.getPegasusBag();
mIPEng = null;
mLogger.logEventCompletion();
//intialize the deployment engine
//requried to setup the TC with the deployed worker package
//executable locations
DeployWorkerPackage deploy = DeployWorkerPackage.loadDeployWorkerPackage( mBag );
deploy.initialize( mReducedDag );
//do the node cluster
if( mPOptions.getClusteringTechnique() != null ){
mLogger.logEventStart( LoggingKeys.EVENT_PEGASUS_CLUSTER, LoggingKeys.DAX_ID, abstractWFName );
mNodeCollapser = new NodeCollapser( mBag );
try{
mReducedDag = mNodeCollapser.cluster( mReducedDag );
}
catch ( Exception e ){
throw new RuntimeException( message, e );
}
mNodeCollapser = null;
mLogger.logEventCompletion();
}
message = "Grafting transfer nodes in the workflow";
PlannerCache plannerCache = new PlannerCache();
plannerCache.initialize(mBag, mReducedDag);
mLogger.log(message,LogManager.INFO_MESSAGE_LEVEL);
mLogger.logEventStart( LoggingKeys.EVENT_PEGASUS_ADD_TRANSFER_NODES, LoggingKeys.DAX_ID, abstractWFName );
mTransEng = new TransferEngine( mReducedDag,
mBag,
mRedEng.getDeletedJobs(),
mRedEng.getDeletedLeafJobs());
mTransEng.addTransferNodes( mRCBridge , plannerCache );
mTransEng = null;
mRedEng = null;
mLogger.logEventCompletion();
//populate the transient RC into PegasusBag
mBag.add( PegasusBag.PLANNER_CACHE, plannerCache );
//close the connection to RLI explicitly
mRCBridge.closeConnection();
//add the deployment of setup jobs if required
mReducedDag = deploy.addSetupNodes( mReducedDag );
if (mPOptions.generateRandomDirectory()) {
//add the nodes to that create
//random directories at the remote
//execution pools.
message = "Grafting the remote workdirectory creation jobs " +
"in the workflow";
//mLogger.log(message,LogManager.INFO_MESSAGE_LEVEL);
mLogger.logEventStart( LoggingKeys.EVENT_PEGASUS_GENERATE_WORKDIR, LoggingKeys.DAX_ID, abstractWFName );
mCreateEng = new CreateDirectory( mBag );
mCreateEng.addCreateDirectoryNodes( mReducedDag );
mCreateEng = null;
mLogger.logEventCompletion();
}
//add the cleanup nodes in place
if ( mPOptions.getCleanup() == null ||
( mPOptions.getCleanup() != PlannerOptions.CLEANUP_OPTIONS.none &&
mPOptions.getCleanup() != PlannerOptions.CLEANUP_OPTIONS.leaf )){
message = "Adding cleanup jobs in the workflow";
mLogger.logEventStart( LoggingKeys.EVENT_PEGASUS_GENERATE_CLEANUP, LoggingKeys.DAX_ID, abstractWFName );
CleanupEngine cEngine = new CleanupEngine( mBag );
mReducedDag = cEngine.addCleanupJobs( mReducedDag );
mLogger.logEventCompletion();
}
if ( mPOptions.getCleanup() == null ||
mPOptions.getCleanup() != PlannerOptions.CLEANUP_OPTIONS.none ){
//add leaf cleanup nodes both when inplace or leaf cleanup is specified
//PM-150 leaf cleanup nodes to remove directories should take care of this
/*
//for the non pegasus lite case we add the cleanup nodes
//for the worker package.
if( !mProps.executeOnWorkerNode() ){
//add the cleanup of setup jobs if required
mReducedDag = deploy.addCleanupNodesForWorkerPackage( mReducedDag );
}
*/
//PM-150
mLogger.logEventStart( "Adding Leaf Cleanup Jobs", LoggingKeys.DAX_ID, abstractWFName );
mRemoveEng = new RemoveDirectory( mReducedDag, mBag, this.mPOptions.getSubmitDirectory() );
mReducedDag = mRemoveEng.addRemoveDirectoryNodes(mReducedDag);
mLogger.logEventCompletion();
mRemoveEng = null;
}
/* PM-714. The approach does not scale for the planner performace test case.
mLogger.logEventStart( "workflow.prune", LoggingKeys.DAX_ID, abstractWFName );
ReduceEdges p = new ReduceEdges();
p.reduce(mReducedDag);
mLogger.logEventCompletion();
*/
mLogger.logEventCompletion();
return mReducedDag;
}
/**
* Returns the cleanup dag for the concrete dag.
*
* @return the cleanup dag if the random dir is given.
* null otherwise.
*/
public ADag getCleanupDAG(){
return mCleanupDag;
}
/**
* Returns the bag of intialization objects.
*
* @return PegasusBag
*/
public PegasusBag getPegasusBag(){
return mBag;
}
/**
* Unmarks the arguments , that are tagged in the DaxParser. At present there are
* no tagging.
*
* @deprecated
*/
private void unmarkArgs() {
/*Enumeration e = mReducedDag.vJobSubInfos.elements();
while(e.hasMoreElements()){
SubInfo sub = (SubInfo)e.nextElement();
sub.strargs = new String(removeMarkups(sub.strargs));
}*/
}
/**
* A small helper method that displays the contents of a Set in a String.
*
* @param s the Set whose contents need to be displayed
* @param delim The delimited between the members of the set.
* @return String
*/
public String setToString(Set s, String delim) {
StringBuffer sb = new StringBuffer();
for( Iterator it = s.iterator(); it.hasNext(); ) {
sb.append( (String) it.next() ).append( delim );
}
String result = sb.toString();
result = (result.length() > 0) ?
result.substring(0, result.lastIndexOf(delim)) :
result;
return result;
}
private void copyCatalogFiles(SiteStore siteStore, TransformationCatalog transformationCatalog, ReplicaCatalogBridge replicaBridge, File directory) {
Set<File> sources = new LinkedHashSet();
sources.addAll( replicaBridge.getReplicaFileSources());
File tc = transformationCatalog.getFileSource();
if( tc != null ){
sources.add( tc );
}
File sc = siteStore.getFileSource();
if( sc != null ){
sources.add( sc );
}
if( !directory.exists() ){
directory.mkdir();
}
for( File source : sources ){
File copiedFile = null;
String failureReason = null;
try {
copiedFile = FileUtils.copy( source , directory);
mLogger.log( "Copied " + source + " to directory " + directory ,
LogManager.DEBUG_MESSAGE_LEVEL );
} catch (IOException ex) {
failureReason = ex.getMessage();
}
if( copiedFile == null ){
mLogger.log( "Unable to copy file " + source + " to directory " + directory + " because " + failureReason,
LogManager.WARNING_MESSAGE_LEVEL );
}
}
}
}