/** * Copyright 2007-2008 University Of Southern California * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package edu.isi.pegasus.planner.namespace; import edu.isi.pegasus.common.logging.LogManager; import edu.isi.pegasus.planner.catalog.classes.Profiles; import edu.isi.pegasus.planner.classes.Profile; import edu.isi.pegasus.planner.common.PegasusProperties; import java.util.Iterator; import java.util.Map; import java.util.TreeMap; /** * This profile namespace is the placeholder for the keys that go into the .dag * file . Keys like RETRY that trigger retries in dagman in the event of a job * failing would go in here. * All the keys stored in it are in UPPERCASE irrespective of the case specified * by the user in the various catalogs. To specify a post script or a pre script * use POST and PRE keys. * * @author Karan Vahi * @author Gaurang Mehta * @version $Revision$ */ public class Dagman extends Namespace { /** * The name of the namespace that this class implements. */ public static final String NAMESPACE_NAME = Profile.DAGMAN; /** * The name of the key that determines what post script is to be invoked * when the job completes. */ public static final String POST_SCRIPT_KEY = "POST"; /** * The name of the key that determines the arguments that need to be passed * to the postscript. */ public static final String POST_SCRIPT_ARGUMENTS_KEY = "POST.ARGUMENTS"; /** * The key prefix that determines the path to a postscript */ public static final String POST_SCRIPT_PATH_PREFIX = "POST.PATH"; /** * The key prefix that determines the path to a postscript */ public static final String POST_SCRIPT_SCOPE_KEY = "POST.SCOPE"; /** * The default value for the arguments passed to postscript */ public static final String DEFAULT_POST_SCRIPT_ARGUMENTS_KEY_VALUE = ""; /** * The name of the key that determines what pre script is to be invoked * when the job is run. */ public static final String PRE_SCRIPT_KEY = "PRE"; /** * The name of the key that determines the arguments that need to be passed * to the postscript. */ public static final String PRE_SCRIPT_ARGUMENTS_KEY = "PRE.ARGUMENTS"; /** * The name of the key that determines the file on the submit host on * which postscript is to be invoked. */ public static final String OUTPUT_KEY = "OUTPUT"; /** * The default value for the arguments passed to prescript */ public static final String DEFAULT_PRE_SCRIPT_ARGUMENTS_KEY_VALUE = ""; /** * The name of the key that determines how many times DAGMAN should be * retrying the job. */ public static final String RETRY_KEY = "RETRY"; /** * The default value for the JOB Retries */ public static final String DEFAULT_RETRY_VALUE = "1"; /** * The name of the key that determines the category to which the job * belongs to. */ public static final String CATEGORY_KEY = "CATEGORY"; /** * The name of the key that determines the priority a job is assigned. */ public static final String PRIORITY_KEY = "PRIORITY"; /** * The name of the key that indicates the path to the corresponding * submit file for the job. */ public static final String JOB_KEY = "JOB"; /** * The name of the key that indicates the path to the external subdag */ public static final String SUBDAG_EXTERNAL_KEY = "SUBDAG EXTERNAL"; /** * The name of the key that indicates the directory in which the * DAG has to be execute */ public static final String DIRECTORY_EXTERNAL_KEY = "DIR"; /** * The name of the key that indicates the NOOP key */ public static final String NOOP_KEY = "NOOP"; /** * The key name for the post script that is put in the .dag file. */ private static final String POST_SCRIPT_REPLACEMENT_KEY = "SCRIPT POST"; /** * The key name for the pre script that is put in the .dag file. */ private static final String PRE_SCRIPT_REPLACEMENT_KEY = "SCRIPT PRE"; /** * The prefix for the max keys */ public static final String MAX_KEYS_PREFIX = "MAX"; /** * The key name for max pre setting for dagman */ public static final String MAXPRE_KEY = "MAXPRE"; /** * The key name for max post setting for dagman */ public static final String MAXPOST_KEY = "MAXPOST"; /** * The key name for max idle setting for dagman */ public static final String MAXIDLE_KEY = "MAXIDLE"; /** * The key name for max jobs setting for dagman */ public static final String MAXJOBS_KEY = "MAXJOBS"; /** * The key name for triggering a job return code to abort the DAG * ABORT-DAG-ON JobName AbortExitValue [RETURN DAGReturnValue] */ public static final String ABORT_DAG_ON_KEY = "ABORT-DAG-ON"; /** * To associate any variables that you want to reference in the job * submit file */ public static final String VARS_KEY = "VARS"; /** * Determines whether a key is category related or not. * * @param key the key in question * * @return */ public static boolean categoryRelatedKey(String key) { boolean result = true; int dotIndex = -1; if( (dotIndex = key.indexOf( "." )) != -1 && dotIndex != key.length() - 1 ){ //the key has a . in it if( key.equals( Dagman.POST_SCRIPT_ARGUMENTS_KEY) || key.equals( Dagman.POST_SCRIPT_SCOPE_KEY) || key.equals( Dagman.PRE_SCRIPT_ARGUMENTS_KEY ) || key.startsWith( Dagman.POST_SCRIPT_PATH_PREFIX) ){ //these are note category related keys return !result; } } else{ return !result; } return result; } /** * The name of the job (jobname) to which the profiles for this namespace * belong. * * @see org.griphyn.cPlanner.classes.SubInfo#jobName */ private String mJobName; /** * The name of the implementing namespace. It should be one of the valid * namespaces always. * * @see Namespace#isNamespaceValid(String) */ protected String mNamespace; /** * The default constructor. * We always initialize the map, as the map is guarenteed to store at least * the postscript value for a job. */ public Dagman() { mProfileMap = new TreeMap(); mNamespace = NAMESPACE_NAME; mJobName = null; } /** * The overloaded constructor. * * @param mp the initial map containing the profile keys for this namespace. */ public Dagman(Map mp) { this(); mProfileMap = new TreeMap(mp); } /** * The overloaded constructor. * * @param mp the initial map containing the profile keys for this namespace. * @param name name of the job with which these profile keys are associated. */ public Dagman(Map mp, String name) { this( mp ); mJobName = name; } /** * Returns the name of the namespace associated with the profile implementations. * * @return the namespace name. * @see #NAMESPACE_NAME */ public String namespaceName(){ return mNamespace; } /** * It sets the name of the job that is associated with the profiles contained * in this placeholder. * * @param name name of the job with which these profile keys are associated. */ public void setJobName(String name){ mJobName = name; } /** * Constructs a new element of the format (key=value). * The underlying map is allocated memory in the constructors always. * All the keys are converted to UPPER CASE before storing. * * @param key is the left-hand-side * @param value is the right hand side */ public void construct(String key, String value) { //convert to uppercase the key mProfileMap.put(key.toUpperCase(), value); } /** * This checks the whether a key value pair specified is valid in the current * namespace or not by calling the checkKey function and then on the basis of * the values returned puts them into the associated map in the class. * * @param key key that needs to be checked in the namespace for validity. * @param value value of the key * */ public void checkKeyInNS(String key, String value){ //convert key to lower case key = key.toUpperCase(); //special handling for category related keys if( categoryRelatedKey( key ) ){ //category related key is ignored mLogger.log( "Dagman category related key cannot be associated with job " + key, LogManager.DEBUG_MESSAGE_LEVEL ); return; } int rslVal = checkKey(key,value); switch (rslVal){ case Namespace.MALFORMED_KEY: //key is malformed ignore malformedKey(key,value); break; case Namespace.NOT_PERMITTED_KEY: notPermitted(key); break; case Namespace.UNKNOWN_KEY: unknownKey(key, value); break; case Namespace.VALID_KEY: construct(key, value); break; case Namespace.DEPRECATED_KEY: deprecatedKey(key,value); break; case Namespace.EMPTY_KEY: emptyKey( key ); break; } } /** * This checks whether the key passed by the user is valid in the current * namespace or not. All keys are assumed valid currently. * * @param key (left hand side) * @param value (right hand side) * * @return Namespace.VALID_KEY * */ public int checkKey(String key, String value) { //all are valid because of certain keys //are defined in SCRIPT POST, that needs //to be corrected int res = 0; if (key == null || key.length() < 2 || value == null || value.length() < 2) { res = MALFORMED_KEY ; } switch (key.charAt(0)) { case 'A': if ( key.compareTo( Dagman.ABORT_DAG_ON_KEY ) == 0 ){ res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'C': if ( key.compareTo( Dagman.CATEGORY_KEY ) == 0 ){ res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'D': if( key.compareTo( Dagman.DIRECTORY_EXTERNAL_KEY) == 0 ){ res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'J': if (key.compareTo(Dagman.JOB_KEY) == 0) { res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'M': if( key.startsWith( MAX_KEYS_PREFIX ) ){ res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'N': if( key.startsWith( NOOP_KEY ) ){ res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'O': if (key.compareTo(Dagman.OUTPUT_KEY) == 0) { res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'P': if ( ( key.compareTo(Dagman.POST_SCRIPT_KEY) == 0) || ( key.compareTo(Dagman.POST_SCRIPT_ARGUMENTS_KEY) == 0)|| ( key.compareTo(Dagman.PRE_SCRIPT_KEY) == 0) || ( key.compareTo(Dagman.PRE_SCRIPT_ARGUMENTS_KEY) == 0 ) || ( key.compareTo(Dagman.POST_SCRIPT_SCOPE_KEY) == 0 ) || ( key.compareTo( Dagman.PRIORITY_KEY) == 0 ) || ( key.startsWith( Dagman.POST_SCRIPT_PATH_PREFIX ) ) ) { res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'R': if (key.compareTo(Dagman.RETRY_KEY) == 0) { res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'S': if (key.compareTo( Dagman.SUBDAG_EXTERNAL_KEY ) == 0) { res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; case 'V': if (key.compareTo(Dagman.VARS_KEY) == 0) { res = VALID_KEY; } else { res = NOT_PERMITTED_KEY; } break; default: res = NOT_PERMITTED_KEY; } return res; } /** * Returns the path to the postscript of a particular type * * @param type type of postscript * * @return the path */ public String getPOSTScriptPath( String type ){ StringBuffer property = new StringBuffer(); property.append( Dagman.POST_SCRIPT_PATH_PREFIX ). append( "." ).append( type.toUpperCase() ); return (String) this.get( property.toString() ); } /** * It puts in the namespace specific information specified in the properties * file into the namespace. The profile information is populated only if the * corresponding key does not exist in the object already. * * @param properties the <code>PegasusProperties</code> object containing * all the properties that the user specified at various * places (like .chimerarc, properties file, command line). * @param pool the pool name where the job is scheduled to run. */ public void checkKeyInNS(PegasusProperties properties, String pool){ //retrieve the relevant profiles from properties //and merge them into the existing. this.assimilate( properties , Profiles.NAMESPACES.dagman ); //check if the arguments for the //post script are specified or not //System.out.println( this.mProfileMap ); if(!this.containsKey(Dagman.POST_SCRIPT_ARGUMENTS_KEY)){ //push in the default arguments for the post script this.checkKeyInNS( Dagman.POST_SCRIPT_ARGUMENTS_KEY, Dagman.DEFAULT_POST_SCRIPT_ARGUMENTS_KEY_VALUE ); } //check if the arguments for the //pre script are specified or not if(!this.containsKey(Dagman.PRE_SCRIPT_ARGUMENTS_KEY)){ //push in the default arguments for the post script this.checkKeyInNS( Dagman.PRE_SCRIPT_ARGUMENTS_KEY, Dagman.DEFAULT_PRE_SCRIPT_ARGUMENTS_KEY_VALUE ); } //what type of postscript needs to be invoked for the job /* if( !this.containsKey( this.POST_SCRIPT_KEY ) ){ //get one from the properties String ps = properties.getPOSTScript(); if( ps != null ){ checkKeyInNS( this.POST_SCRIPT_KEY, properties.getPOSTScript() ); } } */ } /** * Assimilate the profiles in the namespace in a controlled manner. * During assimilation all category related keys are ignored. * * @param profiles the <code>Namespace</code> object containing the profiles. * @param namespace the namespace for which the profiles need to be assimilated. */ public void assimilate( PegasusProperties properties, Profiles.NAMESPACES namespace ){ Namespace profiles = properties.getProfiles( namespace ) ; for ( Iterator it = profiles.getProfileKeyIterator(); it.hasNext(); ){ String key = (String)it.next(); //profiles assimilated from properties have lowest priority if( !this.containsKey(key) ){ this.checkKeyInNS( key, (String)profiles.get( key ) ); } } //profiles in properties have lowest priority //we associate default retry only if user did //not specify in properties if( !this.containsKey( Dagman.RETRY_KEY ) ){ this.construct( RETRY_KEY, DEFAULT_RETRY_VALUE ); } } /** * Merge the profiles in the namespace in a controlled manner. * In case of intersection, the new profile value overrides, the existing * profile value. * * @param profiles the <code>Namespace</code> object containing the profiles. */ public void merge( Namespace profiles ){ //check if we are merging profiles of same type if (!(profiles instanceof Dagman )){ //throw an error throw new IllegalArgumentException( "Profiles mismatch while merging" ); } String key; for ( Iterator it = profiles.getProfileKeyIterator(); it.hasNext(); ){ //construct directly. bypassing the checks! key = (String)it.next(); this.construct( key, (String)profiles.get( key )); } } /** * Converts the contents of the map into the string that can be put in the * Condor file for printing. * * @return the the textual description. */ public String toCondor() { return toString(mJobName); } /** * Converts the contents of the map into the string that can be put in the * Condor file for printing. * * @param name the name of the condor job that contains these variables. * * @return the textual description. */ public String toString(String name) { StringBuffer sb = new StringBuffer(); if(mProfileMap == null){ //no profile keys were stored in here return sb.toString(); } String key = null; for(Iterator it = mProfileMap.keySet().iterator();it.hasNext();){ key = (String) it.next(); //continue to next if the key has to be ignored. if( ignore(key) ){ continue;} append( sb, replacementKey( key ), name, replacementValue( key ) ); // sb.append( replacementKey(key) ).append(" ").append(name). // append(" "). // /*append((String)mProfileMap.get(key))*/ // append( replacementValue(key)). // append("\n"); } //add the ABORT_DAG_ON ky if( this.containsKey( Dagman.ABORT_DAG_ON_KEY) ){ append( sb, Dagman.ABORT_DAG_ON_KEY, name, replacementValue( Dagman.ABORT_DAG_ON_KEY )); } //add the category key in the end if required if( this.containsKey( Dagman.CATEGORY_KEY ) ){ append( sb, replacementKey( Dagman.CATEGORY_KEY ), name, replacementValue( Dagman.CATEGORY_KEY ) ); } //PM-1049 always add VARS dagnode retry append( sb, Dagman.VARS_KEY, name, "+DAGNodeRetry=\"$(RETRY)\"" ); return sb.toString(); } protected StringBuffer append ( StringBuffer sb, String key, String name, String value ){ return sb.append( key ).append(" ").append( name ). append(" ").append( value). append("\n"); } /** * Helper method to decide whether a key has to be ignored or not. * * @param key the key * * @return boolean */ private boolean ignore(String key){ return key.equals( Dagman.POST_SCRIPT_ARGUMENTS_KEY ) || key.equals( Dagman.PRE_SCRIPT_ARGUMENTS_KEY) || key.equals( Dagman.OUTPUT_KEY ) || key.equals( Dagman.CATEGORY_KEY ) || key.equals ( Dagman.POST_SCRIPT_SCOPE_KEY ) || key.startsWith( Dagman.POST_SCRIPT_PATH_PREFIX ) || key.startsWith( Dagman.MAX_KEYS_PREFIX )|| key.startsWith( Dagman.ABORT_DAG_ON_KEY )|| key.equals( Dagman.NOOP_KEY ); } /** * Returns the replacement key that needs to be printed in .dag file in * lieu of the key. * * @param key the key * * @return the replacement key. */ private String replacementKey(String key){ String replacement = key; if(key.equalsIgnoreCase(Dagman.POST_SCRIPT_KEY)){ replacement = Dagman.POST_SCRIPT_REPLACEMENT_KEY; } else if(key.equalsIgnoreCase(Dagman.PRE_SCRIPT_KEY)){ replacement = Dagman.PRE_SCRIPT_REPLACEMENT_KEY; } return replacement; } /** * Returns the replacement value that needs to be printed in .dag file for * a key. This helps us tie the post script path to the arguments, and same * for prescript. * * @param key the key * * @return the replacement value */ private String replacementValue(String key){ StringBuffer value = new StringBuffer(); //append the value for the key value.append( (String)mProfileMap.get(key)); if( key.equals( JOB_KEY ) ){ //PM-987 add NOOP key for job if required if( this.containsKey( NOOP_KEY) ){ //value does not matter value.append( " " ).append( NOOP_KEY ); } } //for postscript and prescript in addition put in the arguments. else if(key.equalsIgnoreCase(Dagman.POST_SCRIPT_KEY)){ //append the postscript arguments value.append(" ").append( (String)this.get( Dagman.POST_SCRIPT_ARGUMENTS_KEY) ); //append the output file value.append(" ").append( (String)this.get( Dagman.OUTPUT_KEY ) ); } else if(key.equalsIgnoreCase(Dagman.PRE_SCRIPT_KEY)){ //append the prescript arguments value.append(" "). append( (String)this.get( Dagman.PRE_SCRIPT_ARGUMENTS_KEY)); } return value.toString(); } /** * Returns a copy of the current namespace object. * * @return the Cloned object */ public Object clone() { Dagman ns = (mProfileMap == null) ? new Dagman():new Dagman(this.mProfileMap); ns.mJobName = (mJobName == null)? null : new String(this.mJobName); return ns; } }