package edu.stanford.nlp.parser.lexparser;
import edu.stanford.nlp.util.logging.Redwood;
import edu.stanford.nlp.trees.Tree;
import edu.stanford.nlp.trees.TreeTransformer;
import edu.stanford.nlp.ling.CategoryWordTag;
import java.util.Set;
import java.io.PrintWriter;
import java.io.Serializable;
/**
* Non-language-specific options for training a grammar from a treebank.
* These options are not used at parsing time.
*
* @author Dan Klein
* @author Christopher Manning
*/
public class TrainOptions implements Serializable {
/** A logger for this class */
private static Redwood.RedwoodChannels log = Redwood.channels(TrainOptions.class);
public String trainTreeFile = null; // same for me -- Teg
/* THESE OPTIONS AFFECT ONLY TRAIN TIME */
public TrainOptions() {}
public int trainLengthLimit = 100000;
/** Add all test set trees to training data for PCFG.
* (Currently only supported in FactoredParser main.)
*/
public boolean cheatPCFG = false;
/** Whether to do "horizontal Markovization" (as in ACL 2003 paper).
* False means regular PCFG expansions.
*/
public boolean markovFactor = false;
public int markovOrder = 1;
public boolean hSelSplit = false; // good with true;
public int HSEL_CUT = 10;
/** Whether or not to mark final states in binarized grammar.
* This must be off to get most value out of grammar compaction.
*/
public boolean markFinalStates = true;
/**
* A POS tag has to have been attributed to more than this number of word
* types before it is regarded as an open-class tag. Unknown words will
* only possibly be tagged as open-class tags (unless flexiTag is on).
* If flexiTag is on, unknown words will be able to be tagged any POS for
* which the unseenMap has nonzero count (that is, the tag was seen for
* a new word after unseen signature counting was started).
*/
public int openClassTypesThreshold = 50;
/**
* Start to aggregate signature-tag pairs only for words unseen in the first
* this fraction of the data.
*/
public double fractionBeforeUnseenCounting = 0.5;
/**
* If true, declare early -- leave this on except maybe with markov on.
* @return Whether to do outside factorization in binarization of the grammar
*/
public boolean outsideFactor() {
return !markovFactor;
}
/**
* This variable controls doing parent annotation of phrasal nodes. Good.
*/
public boolean PA = true;
/**
* This variable controls doing 2 levels of parent annotation. Bad.
*/
public boolean gPA = false;
public boolean postPA = false;
public boolean postGPA = false;
/**
* Only split the "common high KL divergence" parent categories.... Good.
*/
public boolean selectiveSplit = false; //true;
public double selectiveSplitCutOff = 0.0;
public boolean selectivePostSplit = false;
public double selectivePostSplitCutOff = 0.0;
/** Whether, in post-splitting of categories, nodes are annotated with the
* (grand)parent's base category or with its complete subcategorized
* category.
*/
public boolean postSplitWithBaseCategory = false;
/**
* Selective Sister annotation.
*/
public boolean sisterAnnotate = false;
public Set<String> sisterSplitters;
/**
* Mark all unary nodes specially. Good for just PCFG. Bad for factored.
* markUnary affects phrasal nodes. A value of 0 means to do nothing;
* a value of 1 means to mark the parent (higher) node of a unary rewrite.
* A value of 2 means to mark the child (lower) node of a unary rewrie.
* Values of 1 and 2 only apply if the child (lower) node is phrasal.
* (A value of 1 is better than 2 in combos.) A value of 1 corresponds
* to the old boolean -unary flag.
*/
public int markUnary = 0;
/** Mark POS tags which are the sole member of their phrasal constituent.
* This is like markUnary=2, applied to POS tags.
*/
public boolean markUnaryTags = false;
/**
* Mark all pre-preterminals (also does splitBaseNP: don't need both)
*/
public boolean splitPrePreT = false;
/**
* Parent annotation on tags. Good (for PCFG?)
*/
public boolean tagPA = false;//true;
/**
* Do parent annotation on tags selectively. Neutral, but less splits.
*/
public boolean tagSelectiveSplit = false;
public double tagSelectiveSplitCutOff = 0.0;
public boolean tagSelectivePostSplit = false;
public double tagSelectivePostSplitCutOff = 0.0;
/**
* Right edge is right-recursive (X << X) Bad. (NP only is good)
*/
public boolean rightRec = false;//true;
/**
* Left edge is right-recursive (X << X) Bad.
*/
public boolean leftRec = false;
/**
* Promote/delete punctuation like Collins. Bad (!)
*/
public boolean collinsPunc = false;
/**
* Set the splitter strings. These are a set of parent and/or grandparent
* annotated categories which should be split off.
*/
public Set<String> splitters;
public Set postSplitters;
public Set<String> deleteSplitters;
/**
* Just for debugging: check that your tree transforms work correctly. This
* will print the transformations of the first printTreeTransformations trees.
*/
public int printTreeTransformations = 0;
public PrintWriter printAnnotatedPW;
public PrintWriter printBinarizedPW;
// todo [cdm nov 2012]: At present this does nothing. It should print the list of all states of a grammar it trains
// Maybe just make it an anytime option and print it at the same time that verbose printing of tags is done?
public boolean printStates = false;
/** How to compact grammars as FSMs.
* 0 = no compaction [uses makeSyntheticLabel1],
* 1 = no compaction but use label names that wrap from right to left in binarization [uses makeSyntheticLabel2],
* 2 = wrapping labels and materialize unary at top rewriting passive to active,
* 3 = ExactGrammarCompactor,
* 4 = LossyGrammarCompactor,
* 5 = CategoryMergingGrammarCompactor.
* (May 2007 CDM note: options 4 and 5 don't seem to be functioning sensibly. 0, 1, and 3
* seem to be the 'good' options. 2 is only useful as input to 3. There seems to be
* no reason not to use 0, despite the default.)
*/
public int compactGrammar = 3; // exact compaction on by default
public boolean leftToRight = false; // whether to binarize left to right or head out
public int compactGrammar() {
if (markovFactor) {
return compactGrammar;
}
return 0;
}
public boolean noTagSplit = false;
/**
* CHANGE ANYTHING BELOW HERE AT YOUR OWN RISK
*/
/**
* Enables linear rule smoothing during grammar extraction
* but before grammar compaction. The alpha term is the same
* as that described in Petrov et al. (2006), and has range [0,1].
*/
public boolean ruleSmoothing = false;
public double ruleSmoothingAlpha = 0.0;
/**
* TODO wsg2011: This is the old grammar smoothing parameter that no
* longer does anything in the parser. It should be removed.
*/
public boolean smoothing = false;
/* public boolean factorOut = false;
public boolean rightBonus = false;
public boolean brokenDep = false;*/
/** Discounts the count of BinaryRule's (only, apparently) in training data. */
public double ruleDiscount = 0.0;
//public boolean outsideFilter = false;
public boolean printAnnotatedRuleCounts = false;
public boolean printAnnotatedStateCounts = false;
/** Where to use the basic or split tags in the dependency grammar */
public boolean basicCategoryTagsInDependencyGrammar = false;
/**
* A transformer to use on the training data before any other
* processing step. This is specified by using the -preTransformer
* flag when training the parser. A comma separated list of classes
* will be turned into a CompositeTransformer. This can be used to
* strip subcategories, to run a tsurgeon pattern, or any number of
* other useful operations.
*/
public TreeTransformer preTransformer = null;
/**
* A set of files to use as extra information in the lexicon. This
* can provide tagged words which are not part of trees
*/
public String taggedFiles = null;
/**
* Use the method reported by Berkeley for splitting and recombining
* states. This is an experimental and still in development
* reimplementation of that work.
*/
public boolean predictSplits = false;
/**
* If we are predicting splits, we loop this many times
*/
public int splitCount = 1;
/**
* If we are predicting splits, we recombine states at this rate every loop
*/
public double splitRecombineRate = 0.0;
/**
* When binarizing trees, don't annotate the labels with anything
*/
public boolean simpleBinarizedLabels = false;
/**
* When binarizing trees, don't binarize trees with two children.
* Only applies when using inside markov binarization for now.
*/
public boolean noRebinarization = false;
/**
* If the training algorithm allows for parallelization, how many
* threads to use
*/
public int trainingThreads = 1;
/**
* When training the DV parsing method, how many of the top K trees
* to analyze from the underlying parser
*/
static public final int DEFAULT_K_BEST = 100;
public int dvKBest = DEFAULT_K_BEST;
/**
* When training a parsing method where the training has a (max)
* number of iterations, how many iterations to loop
*/
static public final int DEFAULT_TRAINING_ITERATIONS = 40;
public int trainingIterations = DEFAULT_TRAINING_ITERATIONS;
/**
* When training using batches of trees, such as in the DVParser,
* how many trees to use in one batch
*/
static public final int DEFAULT_BATCH_SIZE = 25;
public int batchSize = DEFAULT_BATCH_SIZE;
/**
* regularization constant
*/
public static final double DEFAULT_REGCOST = 0.0001;
public double regCost = DEFAULT_REGCOST;
/**
* When training the DV parsing method, how many iterations to loop
* for one batch of trees
*/
static public final int DEFAULT_QN_ITERATIONS_PER_BATCH = 1;
public int qnIterationsPerBatch = DEFAULT_QN_ITERATIONS_PER_BATCH;
/**
* When training the DV parsing method, how many estimates to keep
* for the qn approximation.
*/
public int qnEstimates = 15;
/**
* When training the DV parsing method, the tolerance to use if we
* want to stop qn early
*/
public double qnTolerance = 15;
/**
* If larger than 0, the parser may choose to output debug information
* every X seconds, X iterations, or some other similar metric
*/
public int debugOutputFrequency = 0;
public long randomSeed = 0;
public static final double DEFAULT_LEARNING_RATE = 0.1;
/**
* How fast to learn (can mean different things for different algorithms)
*/
public double learningRate = DEFAULT_LEARNING_RATE;
public static final double DEFAULT_DELTA_MARGIN = 0.1;
/**
* How much to penalize the wrong trees for how different they are
* from the gold tree when training
*/
public double deltaMargin = DEFAULT_DELTA_MARGIN;
/**
* Whether or not to build an unknown word vector specifically for numbers
*/
public boolean unknownNumberVector = true;
/**
* Whether or not to handle unknown dashed words by taking the last part
*/
public boolean unknownDashedWordVectors = true;
/**
* Whether or not to build an unknown word vector for words with caps in them
*/
public boolean unknownCapsVector = true;
/**
* Make the dv model as simple as possible
*/
public boolean dvSimplifiedModel = false;
/**
* Whether or not to build an unknown word vector to match Chinese years
*/
public boolean unknownChineseYearVector = true;
/**
* Whether or not to build an unknown word vector to match Chinese numbers
*/
public boolean unknownChineseNumberVector = true;
/**
* Whether or not to build an unknown word vector to match Chinese percentages
*/
public boolean unknownChinesePercentVector = true;
public static final double DEFAULT_SCALING_FOR_INIT = 0.5;
/**
* How much to scale certain parameters when initializing models.
* For example, the DVParser uses this to rescale its initial
* matrices.
*/
public double scalingForInit = DEFAULT_SCALING_FOR_INIT;
public int maxTrainTimeSeconds = 0;
public static final String DEFAULT_UNK_WORD = "*UNK*";
/**
* Some models will use external data sources which contain
* information about unknown words. This variable is a way to
* provide the name of the unknown word in the external data source.
*/
public String unkWord = DEFAULT_UNK_WORD;
/**
* Whether or not to lowercase word vectors
*/
public boolean lowercaseWordVectors = false;
public enum TransformMatrixType {
DIAGONAL, RANDOM, OFF_DIAGONAL, RANDOM_ZEROS
}
public TransformMatrixType transformMatrixType = TransformMatrixType.DIAGONAL;
/**
* Specifically for the DVModel, uses words on either side of a
* context when combining constituents. Gives perhaps a microscopic
* improvement in performance but causes a large slowdown.
*/
public boolean useContextWords = false;
/**
* Do we want a model that uses word vectors (such as the DVParser)
* to train those word vectors when training the model?
* <br>
* Note: models prior to 2014-02-13 may have incorrect values in
* this field, as it was originally a compile time constant
*/
public boolean trainWordVectors = true;
public static final int DEFAULT_STALLED_ITERATION_LIMIT = 12;
/**
* How many iterations to allow training to stall before taking the
* best model, if training in an iterative manner
*/
public int stalledIterationLimit = DEFAULT_STALLED_ITERATION_LIMIT;
/** Horton-Strahler number/dimension (Maximilian Schlund) */
public boolean markStrahler;
public void display() {
log.info(toString());
}
@Override
public String toString() {
StringBuilder result = new StringBuilder();
result.append("Train parameters:\n");
result.append(" smooth=" + smoothing + "\n");
result.append(" PA=" + PA + "\n");
result.append(" GPA=" + gPA + "\n");
result.append(" selSplit=" + selectiveSplit + "\n");
result.append(" (" + selectiveSplitCutOff + ((deleteSplitters != null) ? ("; deleting " + deleteSplitters): "") + ")" + "\n");
result.append(" mUnary=" + markUnary + "\n");
result.append(" mUnaryTags=" + markUnaryTags + "\n");
result.append(" sPPT=" + splitPrePreT + "\n");
result.append(" tagPA=" + tagPA + "\n");
result.append(" tagSelSplit=" + tagSelectiveSplit + " (" + tagSelectiveSplitCutOff + ")" + "\n");
result.append(" rightRec=" + rightRec + "\n");
result.append(" leftRec=" + leftRec + "\n");
result.append(" collinsPunc=" + collinsPunc + "\n");
result.append(" markov=" + markovFactor + "\n");
result.append(" mOrd=" + markovOrder + "\n");
result.append(" hSelSplit=" + hSelSplit + " (" + HSEL_CUT + ")" + "\n");
result.append(" compactGrammar=" + compactGrammar() + "\n");
result.append(" postPA=" + postPA + "\n");
result.append(" postGPA=" + postGPA + "\n");
result.append(" selPSplit=" + selectivePostSplit + " (" + selectivePostSplitCutOff + ")" + "\n");
result.append(" tagSelPSplit=" + tagSelectivePostSplit + " (" + tagSelectivePostSplitCutOff + ")" + "\n");
result.append(" postSplitWithBase=" + postSplitWithBaseCategory + "\n");
result.append(" fractionBeforeUnseenCounting=" + fractionBeforeUnseenCounting + "\n");
result.append(" openClassTypesThreshold=" + openClassTypesThreshold + "\n");
result.append(" preTransformer=" + preTransformer + "\n");
result.append(" taggedFiles=" + taggedFiles + "\n");
result.append(" predictSplits=" + predictSplits + "\n");
result.append(" splitCount=" + splitCount + "\n");
result.append(" splitRecombineRate=" + splitRecombineRate + "\n");
result.append(" simpleBinarizedLabels=" + simpleBinarizedLabels + "\n");
result.append(" noRebinarization=" + noRebinarization + "\n");
result.append(" trainingThreads=" + trainingThreads + "\n");
result.append(" dvKBest=" + dvKBest + "\n");
result.append(" trainingIterations=" + trainingIterations + "\n");
result.append(" batchSize=" + batchSize + "\n");
result.append(" regCost=" + regCost + "\n");
result.append(" qnIterationsPerBatch=" + qnIterationsPerBatch + "\n");
result.append(" qnEstimates=" + qnEstimates + "\n");
result.append(" qnTolerance=" + qnTolerance + "\n");
result.append(" debugOutputFrequency=" + debugOutputFrequency + "\n");
result.append(" randomSeed=" + randomSeed + "\n");
result.append(" learningRate=" + learningRate + "\n");
result.append(" deltaMargin=" + deltaMargin + "\n");
result.append(" unknownNumberVector=" + unknownNumberVector + "\n");
result.append(" unknownDashedWordVectors=" + unknownDashedWordVectors + "\n");
result.append(" unknownCapsVector=" + unknownCapsVector + "\n");
result.append(" unknownChineseYearVector=" + unknownChineseYearVector + "\n");
result.append(" unknownChineseNumberVector=" + unknownChineseNumberVector + "\n");
result.append(" unknownChinesePercentVector=" + unknownChinesePercentVector + "\n");
result.append(" dvSimplifiedModel=" + dvSimplifiedModel + "\n");
result.append(" scalingForInit=" + scalingForInit + "\n");
result.append(" maxTrainTimeSeconds=" + maxTrainTimeSeconds + "\n");
result.append(" unkWord=" + unkWord + "\n");
result.append(" lowercaseWordVectors=" + lowercaseWordVectors + "\n");
result.append(" transformMatrixType=" + transformMatrixType + "\n");
result.append(" useContextWords=" + useContextWords + "\n");
result.append(" trainWordVectors=" + trainWordVectors + "\n");
result.append(" stalledIterationLimit=" + stalledIterationLimit + "\n");
result.append(" markStrahler=" + markStrahler + "\n");
return result.toString();
}
public static void printTrainTree(PrintWriter pw, String message, Tree t) {
PrintWriter myPW;
if (pw == null) {
myPW = new PrintWriter(System.out, true);
} else {
myPW = pw;
}
if (message != null && pw == null) {
// hard coded to not print message if using file output!
myPW.println(message);
}
// TODO FIXME: wtf is this shit
boolean previousState = CategoryWordTag.printWordTag;
CategoryWordTag.printWordTag = false;
t.pennPrint(myPW);
CategoryWordTag.printWordTag = previousState;
}
private static final long serialVersionUID = 72571349843538L;
} // end class Train