//
// StanfordCoreNLP -- a suite of NLP tools
// Copyright (c) 2009-2011 The Board of Trustees of
// The Leland Stanford Junior University. All Rights Reserved.
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//
// For more information, bug reports, fixes, contact:
// Christopher Manning
// Dept of Computer Science, Gates 1A
// Stanford CA 94305-9010
// USA
//
package edu.stanford.nlp.dcoref;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.HashSet;
import java.util.TreeSet;
import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import edu.stanford.nlp.pipeline.DefaultPaths;
import edu.stanford.nlp.classify.LogisticClassifier;
import edu.stanford.nlp.dcoref.CorefChain.CorefMention;
import edu.stanford.nlp.dcoref.Dictionaries.MentionType;
import edu.stanford.nlp.dcoref.ScorerBCubed.BCubedType;
import edu.stanford.nlp.dcoref.sievepasses.DeterministicCorefSieve;
import edu.stanford.nlp.dcoref.sievepasses.ExactStringMatch;
import edu.stanford.nlp.io.IOUtils;
import edu.stanford.nlp.io.RuntimeIOException;
import edu.stanford.nlp.io.StringOutputStream;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.stats.ClassicCounter;
import edu.stanford.nlp.stats.Counter;
import edu.stanford.nlp.trees.Tree;
import edu.stanford.nlp.util.*;
import edu.stanford.nlp.util.logging.NewlineLogFormatter;
/**
* Multi-pass Sieve coreference resolution system (see EMNLP 2010 paper).
* <p>
* The main entry point for API is coref(Document document).
* The output is a map from CorefChain ID to corresponding CorefChain.
*
* @author Jenny Finkel
* @author Mihai Surdeanu
* @author Karthik Raghunathan
* @author Heeyoung Lee
* @author Sudarshan Rangarajan
*/
public class SieveCoreferenceSystem {
/** A logger for this class. Still uses j.u.l currently. */
public static final Logger logger = Logger.getLogger(SieveCoreferenceSystem.class.getName());
/**
* If true, we score the output of the given test document
* Assumes gold annotations are available
*/
private final boolean doScore;
/**
* If true, we do post processing.
*/
private final boolean doPostProcessing;
/**
* maximum sentence distance between two mentions for resolution (-1: no constraint on distance)
*/
private final int maxSentDist;
/**
* automatically set by looking at sieves
*/
private final boolean useSemantics;
/**
* Singleton predictor from Recasens, de Marneffe, and Potts (NAACL 2013)
*/
private final boolean useSingletonPredictor;
/** flag for replicating CoNLL result */
private final boolean replicateCoNLL;
/** Path for the official CoNLL scorer */
public final String conllMentionEvalScript;
/** flag for optimizing ordering of sieves */
private final boolean optimizeSieves;
/** Constraints on sieve order */
private List<Pair<Integer,Integer>> sievesKeepOrder;
/** Final score to use for sieve optimization (default is pairwise.Precision) */
private final String optimizeScoreType;
/** More useful break down of optimizeScoreType */
private final boolean optimizeConllScore;
private final String optimizeMetricType;
private final CorefScorer.SubScoreType optimizeSubScoreType;
/**
* Array of sieve passes to be used in the system
* Ordered from highest precision to lowest!
*/
/** Not final because may change when running optimize sieve ordering but otherwise should stay fixed */
private /*final */DeterministicCorefSieve [] sieves;
private /*final*/ String [] sieveClassNames;
/**
* Dictionaries of all the useful goodies (gender, animacy, number etc. lists)
*/
private final Dictionaries dictionaries;
/**
* Semantic knowledge: WordNet
*/
private final Semantics semantics;
private LogisticClassifier<String, String> singletonPredictor;
// Below are member variables used for scoring (not thread safe)
/** Current sieve index */
private int currentSieve;
/** counter for links in passes ({@code Pair<correct links, total links>}) */
private List<Pair<Integer, Integer>> linksCountInPass;
/** Scores for each pass */
private List<CorefScorer> scorePairwise;
private List<CorefScorer> scoreBcubed;
private List<CorefScorer> scoreMUC;
private List<CorefScorer> scoreSingleDoc;
/** Additional scoring stats */
private int additionalCorrectLinksCount;
private int additionalLinksCount;
public SieveCoreferenceSystem(Properties props) throws Exception {
// initialize required fields
currentSieve = -1;
//
// construct the sieve passes
//
String sievePasses = props.getProperty(Constants.SIEVES_PROP, Constants.SIEVEPASSES);
sieveClassNames = sievePasses.trim().split(",\\s*");
sieves = new DeterministicCorefSieve[sieveClassNames.length];
for(int i = 0; i < sieveClassNames.length; i ++){
sieves[i] = (DeterministicCorefSieve) Class.forName("edu.stanford.nlp.dcoref.sievepasses."+sieveClassNames[i]).getConstructor().newInstance();
sieves[i].init(props);
}
//
// create scoring framework
//
doScore = Boolean.parseBoolean(props.getProperty(Constants.SCORE_PROP, "false"));
//
// setting post processing
//
doPostProcessing = Boolean.parseBoolean(props.getProperty(Constants.POSTPROCESSING_PROP, "false"));
//
// setting singleton predictor
//
useSingletonPredictor = Boolean.parseBoolean(props.getProperty(Constants.SINGLETON_PROP, "true"));
//
// setting maximum sentence distance between two mentions for resolution (-1: no constraint on distance)
//
maxSentDist = Integer.parseInt(props.getProperty(Constants.MAXDIST_PROP, "-1"));
//
// set useWordNet
//
useSemantics = sievePasses.contains("AliasMatch") || sievePasses.contains("LexicalChainMatch");
// flag for replicating CoNLL result
replicateCoNLL = Boolean.parseBoolean(props.getProperty(Constants.REPLICATECONLL_PROP, "false"));
conllMentionEvalScript = props.getProperty(Constants.CONLL_SCORER, Constants.conllMentionEvalScript);
// flag for optimizing sieve ordering
optimizeSieves = Boolean.parseBoolean(props.getProperty(Constants.OPTIMIZE_SIEVES_PROP, "false"));
optimizeScoreType = props.getProperty(Constants.OPTIMIZE_SIEVES_SCORE_PROP, "pairwise.Precision");
// Break down of the optimize score type
String[] validMetricTypes = { "muc", "pairwise", "bcub", "ceafe", "ceafm", "combined" };
String[] parts = optimizeScoreType.split("\\.");
optimizeConllScore = parts.length > 2 && "conll".equalsIgnoreCase(parts[2]);
optimizeMetricType = parts[0];
boolean optimizeMetricTypeOk = false;
for (String validMetricType : validMetricTypes) {
if (validMetricType.equalsIgnoreCase(optimizeMetricType)) {
optimizeMetricTypeOk = true;
break;
}
}
if (!optimizeMetricTypeOk) {
throw new IllegalArgumentException("Invalid metric type for " +
Constants.OPTIMIZE_SIEVES_SCORE_PROP + " property: " + optimizeScoreType);
}
optimizeSubScoreType = CorefScorer.SubScoreType.valueOf(parts[1]);
if (optimizeSieves) {
String keepSieveOrder = props.getProperty(Constants.OPTIMIZE_SIEVES_KEEP_ORDER_PROP);
if (keepSieveOrder != null) {
String[] orderings = keepSieveOrder.split("\\s*,\\s*");
sievesKeepOrder = new ArrayList<>();
String firstSieveConstraint = null;
String lastSieveConstraint = null;
for (String ordering:orderings) {
// Convert ordering constraints from string
Pair<Integer,Integer> p = fromSieveOrderConstraintString(ordering, sieveClassNames);
// Do initial check of sieves order, can only have one where the first is ANY (< 0), and one where second is ANY (< 0)
if (p.first() < 0 && p.second() < 0) {
throw new IllegalArgumentException("Invalid ordering constraint: " + ordering);
} else if (p.first() < 0) {
if (lastSieveConstraint != null) {
throw new IllegalArgumentException("Cannot have these two ordering constraints: " + lastSieveConstraint + "," + ordering);
}
lastSieveConstraint = ordering;
} else if (p.second() < 0) {
if (firstSieveConstraint != null) {
throw new IllegalArgumentException("Cannot have these two ordering constraints: " + firstSieveConstraint + "," + ordering);
}
firstSieveConstraint = ordering;
}
sievesKeepOrder.add(p);
}
}
}
if(doScore){
initScorers();
}
//
// load all dictionaries
//
dictionaries = new Dictionaries(props);
semantics = (useSemantics)? new Semantics(dictionaries) : null;
if(useSingletonPredictor){
singletonPredictor = getSingletonPredictorFromSerializedFile(props.getProperty(Constants.SINGLETON_MODEL_PROP, DefaultPaths.DEFAULT_DCOREF_SINGLETON_MODEL));
}
}
public void initScorers() {
linksCountInPass = new ArrayList<>();
scorePairwise = new ArrayList<>();
scoreBcubed = new ArrayList<>();
scoreMUC = new ArrayList<>();
for (String sieveClassName : sieveClassNames) {
scorePairwise.add(new ScorerPairwise());
scoreBcubed.add(new ScorerBCubed(BCubedType.Bconll));
scoreMUC.add(new ScorerMUC());
linksCountInPass.add(new Pair<>(0, 0));
}
}
public boolean doScore() { return doScore; }
public Dictionaries dictionaries() { return dictionaries; }
public Semantics semantics() { return semantics; }
public String sieveClassName(int sieveIndex) {
return (sieveIndex >= 0 && sieveIndex < sieveClassNames.length)? sieveClassNames[sieveIndex]:null; }
/**
* Needs the following properties:
* -props 'Location of coref.properties'
* @throws Exception
*/
public static void main(String[] args) throws Exception {
Properties props = StringUtils.argsToProperties(args);
initializeAndRunCoref(props);
}
/** Returns the name of the log file that this method writes. */
public static String initializeAndRunCoref(Properties props) throws Exception {
String timeStamp = Calendar.getInstance().getTime().toString().replaceAll("\\s", "-").replaceAll(":", "-");
//
// initialize logger
//
String logFileName = props.getProperty(Constants.LOG_PROP, "log.txt");
if (logFileName.endsWith(".txt")) {
logFileName = logFileName.substring(0, logFileName.length()-4) +"_"+ timeStamp+".txt";
} else {
logFileName = logFileName + "_"+ timeStamp+".txt";
}
try {
FileHandler fh = new FileHandler(logFileName, false);
logger.addHandler(fh);
logger.setLevel(Level.FINE);
fh.setFormatter(new NewlineLogFormatter());
} catch (SecurityException | IOException e) {
throw new RuntimeException("Cannot initialize logger!", e);
}
logger.fine(timeStamp);
logger.fine(props.toString());
Constants.printConstants(logger);
// initialize coref system
SieveCoreferenceSystem corefSystem = new SieveCoreferenceSystem(props);
// MentionExtractor extracts MUC, ACE, or CoNLL documents
MentionExtractor mentionExtractor;
if (props.containsKey(Constants.MUC_PROP)){
mentionExtractor = new MUCMentionExtractor(corefSystem.dictionaries, props,
corefSystem.semantics, corefSystem.singletonPredictor);
} else if(props.containsKey(Constants.ACE2004_PROP) || props.containsKey(Constants.ACE2005_PROP)) {
mentionExtractor = new ACEMentionExtractor(corefSystem.dictionaries, props,
corefSystem.semantics, corefSystem.singletonPredictor);
} else if (props.containsKey(Constants.CONLL2011_PROP)) {
mentionExtractor = new CoNLLMentionExtractor(corefSystem.dictionaries, props,
corefSystem.semantics, corefSystem.singletonPredictor);
} else {
throw new RuntimeException("No input file specified!");
}
if (!Constants.USE_GOLD_MENTIONS) {
// Set mention finder
String mentionFinderClass = props.getProperty(Constants.MENTION_FINDER_PROP);
if (mentionFinderClass != null) {
String mentionFinderPropFilename = props.getProperty(Constants.MENTION_FINDER_PROPFILE_PROP);
CorefMentionFinder mentionFinder;
if (mentionFinderPropFilename != null) {
Properties mentionFinderProps = new Properties();
FileInputStream fis = new FileInputStream(mentionFinderPropFilename);
mentionFinderProps.load(fis);
fis.close();
mentionFinder = (CorefMentionFinder) Class.forName(mentionFinderClass).getConstructor(Properties.class).newInstance(mentionFinderProps);
} else {
mentionFinder = (CorefMentionFinder) Class.forName(mentionFinderClass).newInstance();
}
mentionExtractor.setMentionFinder(mentionFinder);
}
if (mentionExtractor.mentionFinder == null) {
logger.warning("No mention finder specified, but not using gold mentions");
}
}
if (corefSystem.optimizeSieves && corefSystem.sieves.length > 1) {
corefSystem.optimizeSieveOrdering(mentionExtractor, props, timeStamp);
}
try {
runAndScoreCoref(corefSystem, mentionExtractor, props, timeStamp);
} catch (Exception ex) {
logger.log(Level.SEVERE, "ERROR in running coreference", ex);
}
logger.info("done");
String endTimeStamp = Calendar.getInstance().getTime().toString().replaceAll("\\s", "-");
logger.fine(endTimeStamp);
return logFileName;
}
public static double runAndScoreCoref(SieveCoreferenceSystem corefSystem,
MentionExtractor mentionExtractor,
Properties props,
String timeStamp) throws Exception
{
// prepare conll output
PrintWriter writerGold = null;
PrintWriter writerPredicted = null;
PrintWriter writerPredictedCoref = null;
String conllOutputMentionGoldFile = null;
String conllOutputMentionPredictedFile = null;
String conllOutputMentionCorefPredictedFile = null;
String conllMentionEvalFile = null;
String conllMentionEvalErrFile = null;
String conllMentionCorefEvalFile = null;
String conllMentionCorefEvalErrFile = null;
if(Constants.PRINT_CONLL_OUTPUT || corefSystem.replicateCoNLL) {
String conllOutput = props.getProperty(Constants.CONLL_OUTPUT_PROP, "conlloutput");
conllOutputMentionGoldFile = conllOutput + "-"+timeStamp+".gold.txt";
conllOutputMentionPredictedFile = conllOutput +"-"+timeStamp+ ".predicted.txt";
conllOutputMentionCorefPredictedFile = conllOutput +"-"+timeStamp+ ".coref.predicted.txt";
conllMentionEvalFile = conllOutput +"-"+timeStamp+ ".eval.txt";
conllMentionEvalErrFile = conllOutput +"-"+timeStamp+ ".eval.err.txt";
conllMentionCorefEvalFile = conllOutput +"-"+timeStamp+ ".coref.eval.txt";
conllMentionCorefEvalErrFile = conllOutput +"-"+timeStamp+ ".coref.eval.err.txt";
logger.info("CONLL MENTION GOLD FILE: " + conllOutputMentionGoldFile);
logger.info("CONLL MENTION PREDICTED FILE: " + conllOutputMentionPredictedFile);
logger.info("CONLL MENTION EVAL FILE: " + conllMentionEvalFile);
if (!Constants.SKIP_COREF) {
logger.info("CONLL MENTION PREDICTED WITH COREF FILE: " + conllOutputMentionCorefPredictedFile);
logger.info("CONLL MENTION WITH COREF EVAL FILE: " + conllMentionCorefEvalFile);
}
writerGold = new PrintWriter(new FileOutputStream(conllOutputMentionGoldFile));
writerPredicted = new PrintWriter(new FileOutputStream(conllOutputMentionPredictedFile));
writerPredictedCoref = new PrintWriter(new FileOutputStream(conllOutputMentionCorefPredictedFile));
}
mentionExtractor.resetDocs();
if (corefSystem.doScore()) {
corefSystem.initScorers();
}
//
// Parse one document at a time, and do single-doc coreference resolution in each.
//
// In one iteration, orderedMentionsBySentence contains a list of all
// mentions in one document. Each mention has properties (annotations):
// its surface form (Word), NER Tag, POS Tag, Index, etc.
//
while(true) {
Document document = mentionExtractor.nextDoc();
if(document==null) break;
if(!props.containsKey(Constants.MUC_PROP)) {
printRawDoc(document, true);
printRawDoc(document, false);
}
printDiscourseStructure(document);
if(corefSystem.doScore()){
document.extractGoldCorefClusters();
}
if(Constants.PRINT_CONLL_OUTPUT || corefSystem.replicateCoNLL) {
// Not doing coref - print conll output here
printConllOutput(document, writerGold, true);
printConllOutput(document, writerPredicted, false);
}
// run mention detection only
if(Constants.SKIP_COREF) {
continue;
}
corefSystem.coref(document); // Do Coreference Resolution
if(corefSystem.doScore()){
//Identifying possible coreferring mentions in the corpus along with any recall/precision errors with gold corpus
corefSystem.printTopK(logger, document, corefSystem.semantics);
logger.fine("pairwise score for this doc: ");
corefSystem.scoreSingleDoc.get(corefSystem.sieves.length-1).printF1(logger);
logger.fine("accumulated score: ");
corefSystem.printF1(true);
logger.fine("\n");
}
if(Constants.PRINT_CONLL_OUTPUT || corefSystem.replicateCoNLL){
printConllOutput(document, writerPredictedCoref, false, true);
}
}
double finalScore = 0;
if(Constants.PRINT_CONLL_OUTPUT || corefSystem.replicateCoNLL) {
writerGold.close();
writerPredicted.close();
writerPredictedCoref.close();
//if(props.containsKey(Constants.CONLL_SCORER)) {
if (corefSystem.conllMentionEvalScript != null) {
// runConllEval(corefSystem.conllMentionEvalScript, conllOutputMentionGoldFile, conllOutputMentionPredictedFile, conllMentionEvalFile, conllMentionEvalErrFile);
String summary = getConllEvalSummary(corefSystem.conllMentionEvalScript, conllOutputMentionGoldFile, conllOutputMentionPredictedFile);
logger.info("\nCONLL EVAL SUMMARY (Before COREF)");
printScoreSummary(summary, logger, false);
if (!Constants.SKIP_COREF) {
// runConllEval(corefSystem.conllMentionEvalScript, conllOutputMentionGoldFile, conllOutputMentionCorefPredictedFile, conllMentionCorefEvalFile, conllMentionCorefEvalErrFile);
summary = getConllEvalSummary(corefSystem.conllMentionEvalScript, conllOutputMentionGoldFile, conllOutputMentionCorefPredictedFile);
logger.info("\nCONLL EVAL SUMMARY (After COREF)");
printScoreSummary(summary, logger, true);
printFinalConllScore(summary);
if (corefSystem.optimizeConllScore) {
finalScore = getFinalConllScore(summary, corefSystem.optimizeMetricType, corefSystem.optimizeSubScoreType.toString());
}
}
}
}
if (!corefSystem.optimizeConllScore && corefSystem.doScore()) {
finalScore = corefSystem.getFinalScore(corefSystem.optimizeMetricType, corefSystem.optimizeSubScoreType);
}
String scoresFile = props.getProperty(Constants.SCORE_FILE_PROP);
if (scoresFile != null) {
PrintWriter pw = IOUtils.getPrintWriter(scoresFile);
pw.println((new DecimalFormat("#.##")).format(finalScore));
pw.close();
}
if (corefSystem.optimizeSieves) {
logger.info("Final reported score for sieve optimization " + corefSystem.optimizeScoreType + " : " + finalScore);
}
return finalScore;
}
/** Run and score coref distributed */
public static void runAndScoreCorefDist(String runDistCmd, Properties props, String propsFile) throws Exception {
PrintWriter pw = IOUtils.getPrintWriter(propsFile);
props.store(pw, null);
pw.close();
/* Run coref job in a distributed manner, score is written to file */
List<String> cmd = new ArrayList<>();
cmd.addAll(Arrays.asList(runDistCmd.split("\\s+")));
cmd.add("-props");
cmd.add(propsFile);
ProcessBuilder pb = new ProcessBuilder(cmd);
// Copy environment variables over
Map<String,String> curEnv = System.getenv();
Map<String,String> pbEnv = pb.environment();
pbEnv.putAll(curEnv);
logger.info("Running distributed coref:" + StringUtils.join(pb.command(), " "));
StringWriter outSos = new StringWriter();
StringWriter errSos = new StringWriter();
PrintWriter out = new PrintWriter(new BufferedWriter(outSos));
PrintWriter err = new PrintWriter(new BufferedWriter(errSos));
SystemUtils.run(pb, out, err);
out.close();
err.close();
String outStr = outSos.toString();
String errStr = errSos.toString();
logger.info("Finished distributed coref: " + runDistCmd + ", props=" + propsFile);
logger.info("Output: " + outStr);
if (errStr.length() > 0) {
logger.info("Error: " + errStr);
}
}
static boolean waitForFiles(File workDir, FileFilter fileFilter, int howMany) throws InterruptedException {
logger.info("Waiting until we see " + howMany + " " + fileFilter + " files in directory " + workDir + "...");
int seconds = 0;
while (true) {
File[] checkFiles = workDir.listFiles(fileFilter);
// we found the required number of .check files
if (checkFiles != null && checkFiles.length >= howMany) {
logger.info("Found " + checkFiles.length + " " + fileFilter + " files. Continuing execution.");
break;
}
// sleep for while before the next check
Thread.sleep(Constants.MONITOR_DIST_CMD_FINISHED_WAIT_MILLIS);
seconds += Constants.MONITOR_DIST_CMD_FINISHED_WAIT_MILLIS / 1000;
if (seconds % 600 == 0) {
double minutes = seconds / 60;
logger.info("Still waiting... " + minutes + " minutes have passed.");
}
}
return true;
}
private static int fromSieveNameToIndex(String sieveName, String[] sieveNames)
{
if ("*".equals(sieveName)) return -1;
for (int i = 0; i < sieveNames.length; i++) {
if (sieveNames[i].equals(sieveName)) {
return i;
}
}
throw new IllegalArgumentException("Invalid sieve name: " + sieveName);
}
private static Pair<Integer,Integer> fromSieveOrderConstraintString(String s, String[] sieveNames)
{
String[] parts = s.split("<");
if (parts.length == 2) {
String first = parts[0].trim();
String second = parts[1].trim();
int a = fromSieveNameToIndex(first, sieveNames);
int b = fromSieveNameToIndex(second, sieveNames);
return new Pair<>(a, b);
} else {
throw new IllegalArgumentException("Invalid sieve ordering constraint: " + s);
}
}
private static String toSieveOrderConstraintString(Pair<Integer,Integer> orderedSieveIndices, String[] sieveNames)
{
String first = (orderedSieveIndices.first() < 0)? "*":sieveNames[orderedSieveIndices.first()];
String second = (orderedSieveIndices.second() < 0)? "*":sieveNames[orderedSieveIndices.second()];
return first + " < " + second;
}
/**
* Given a set of sieves, select an optimal ordering for the sieves
* by iterating over sieves, and selecting the one that gives the best score and
* adding sieves one at a time until no more sieves left
*/
public void optimizeSieveOrdering(MentionExtractor mentionExtractor, Properties props, String timestamp) throws Exception
{
logger.info("=============SIEVE OPTIMIZATION START ====================");
logger.info("Optimize sieves using score: " + optimizeScoreType);
FileFilter scoreFilesFilter = new FileFilter() {
@Override
public boolean accept(File file) {
return file.getAbsolutePath().endsWith(".score");
}
public String toString() {
return ".score";
}
};
Pattern scoreFilePattern = Pattern.compile(".*sieves\\.(\\d+)\\.(\\d+).score");
String runDistributedCmd = props.getProperty(Constants.RUN_DIST_CMD_PROP);
String mainWorkDirPath = props.getProperty(Constants.RUN_DIST_CMD_WORK_DIR, "workdir") + "-" + timestamp + File.separator;
DeterministicCorefSieve[] origSieves = sieves;
String[] origSieveNames = sieveClassNames;
Set<Integer> remainingSieveIndices = Generics.newHashSet();
for (int i = 0; i < origSieves.length; i++) {
remainingSieveIndices.add(i);
}
List<Integer> optimizedOrdering = new ArrayList<>();
while (!remainingSieveIndices.isEmpty()) {
// initialize array of current sieves
int curSievesNumber = optimizedOrdering.size();
sieves = new DeterministicCorefSieve[curSievesNumber+1];
sieveClassNames = new String[curSievesNumber+1];
for (int i = 0; i < curSievesNumber; i++) {
sieves[i] = origSieves[optimizedOrdering.get(i)];
sieveClassNames[i] = origSieveNames[optimizedOrdering.get(i)];
}
logger.info("*** Optimizing Sieve ordering for pass " + curSievesNumber + " ***");
// Get list of sieves that we can pick from for the next sieve
Set<Integer> selectableSieveIndices = new TreeSet<>(remainingSieveIndices);
// Based on ordering constraints remove sieves from options
if (sievesKeepOrder != null) {
for (Pair<Integer,Integer> ko:sievesKeepOrder) {
if (ko.second() < 0) {
if (remainingSieveIndices.contains(ko.first())) {
logger.info("Restrict selection to " + origSieveNames[ko.first()] + " because of constraint " +
toSieveOrderConstraintString(ko, origSieveNames));
selectableSieveIndices = Generics.newHashSet(1);
selectableSieveIndices.add(ko.first());
break;
}
} else if (ko.first() < 0 && remainingSieveIndices.size() > 1) {
if (remainingSieveIndices.contains(ko.second())) {
logger.info("Remove selection " + origSieveNames[ko.second()] + " because of constraint " +
toSieveOrderConstraintString(ko, origSieveNames));
selectableSieveIndices.remove(ko.second());
}
} else if (remainingSieveIndices.contains(ko.first())) {
if (remainingSieveIndices.contains(ko.second())) {
logger.info("Remove selection " + origSieveNames[ko.second()] + " because of constraint " +
toSieveOrderConstraintString(ko, origSieveNames));
selectableSieveIndices.remove(ko.second());
}
}
}
}
if (selectableSieveIndices.isEmpty()) {
throw new RuntimeException("Unable to find sieve ordering to satisfy all ordering constraints!!!!");
}
int selected = -1;
if (selectableSieveIndices.size() > 1) {
// Go through remaining sieves and see how well they do
List<Pair<Double,Integer>> scores = new ArrayList<>();
if (runDistributedCmd != null) {
String workDirPath = mainWorkDirPath + curSievesNumber + File.separator;
File workDir = new File(workDirPath);
workDir.mkdirs();
workDirPath = workDir.getAbsolutePath() + File.separator;
// Start jobs
for (int potentialSieveIndex:selectableSieveIndices) {
String sieveSelectionId = curSievesNumber + "." + potentialSieveIndex;
String jobDirPath = workDirPath + sieveSelectionId + File.separator;
File jobDir = new File(jobDirPath);
jobDir.mkdirs();
Properties newProps = new Properties();
for (String key:props.stringPropertyNames()) {
String value = props.getProperty(key);
value = value.replaceAll("\\$\\{JOBDIR\\}",jobDirPath);
newProps.setProperty(key, value);
}
// try this sieve and see how well it works
sieves[curSievesNumber] = origSieves[potentialSieveIndex];
sieveClassNames[curSievesNumber] = origSieveNames[potentialSieveIndex];
newProps.setProperty(Constants.OPTIMIZE_SIEVES_PROP, "false");
newProps.setProperty(Constants.SCORE_PROP, "true");
newProps.setProperty(Constants.SIEVES_PROP, StringUtils.join(sieveClassNames,","));
newProps.setProperty(Constants.LOG_PROP, jobDirPath + "sieves." + sieveSelectionId + ".log");
newProps.setProperty(Constants.SCORE_FILE_PROP, workDirPath + "sieves." + sieveSelectionId + ".score");
if (Constants.PRINT_CONLL_OUTPUT || replicateCoNLL) {
newProps.setProperty(Constants.CONLL_OUTPUT_PROP, jobDirPath + "sieves." + sieveSelectionId + ".conlloutput");
}
String distCmd = newProps.getProperty(Constants.RUN_DIST_CMD_PROP, runDistributedCmd);
runAndScoreCorefDist(distCmd, newProps, workDirPath + "sieves." + sieveSelectionId + ".props");
}
// Wait for jobs to finish and collect scores
waitForFiles(workDir, scoreFilesFilter,selectableSieveIndices.size());
// Get scores
File[] scoreFiles = workDir.listFiles(scoreFilesFilter);
for (File file:scoreFiles) {
Matcher m = scoreFilePattern.matcher(file.getName());
if (m.matches()) {
int potentialSieveIndex = Integer.parseInt(m.group(2));
String text = IOUtils.slurpFile(file);
double score = Double.parseDouble(text);
// keeps scores so we can select best score and log them
scores.add(new Pair<>(score, potentialSieveIndex));
} else {
throw new RuntimeException("Bad score file name: " + file);
}
}
} else {
for (int potentialSieveIndex:selectableSieveIndices) {
// try this sieve and see how well it works
sieves[curSievesNumber] = origSieves[potentialSieveIndex];
sieveClassNames[curSievesNumber] = origSieveNames[potentialSieveIndex];
logger.info("Trying sieve " + curSievesNumber + "="+ sieveClassNames[curSievesNumber] + ": ");
logger.info(" Trying sieves: " + StringUtils.join(sieveClassNames,","));
double score = runAndScoreCoref(this, mentionExtractor, props, timestamp);
// keeps scores so we can select best score and log them
scores.add(new Pair<>(score, potentialSieveIndex));
logger.info(" Trying sieves: " + StringUtils.join(sieveClassNames,","));
logger.info(" Trying sieves score: " + score);
}
}
// Select bestScore
double bestScore = -1;
for (Pair<Double,Integer> p:scores) {
if (selected < 0 || p.first() > bestScore) {
bestScore = p.first();
selected = p.second();
}
}
// log ordered scores
Collections.sort(scores);
Collections.reverse(scores);
logger.info("Ordered sieves");
for (Pair<Double,Integer> p:scores) {
logger.info("Sieve optimization pass " + curSievesNumber
+ " scores: Sieve=" + origSieveNames[p.second()] + ", score=" + p.first());
}
} else {
// Only one sieve
logger.info("Only one choice for next sieve");
selected = selectableSieveIndices.iterator().next();
}
// log sieve we are adding
sieves[curSievesNumber] = origSieves[selected];
sieveClassNames[curSievesNumber] = origSieveNames[selected];
logger.info("Adding sieve " + curSievesNumber + "="+ sieveClassNames[curSievesNumber] + " to existing sieves: ");
logger.info(" Current Sieves: " + StringUtils.join(sieveClassNames,","));
// select optimal sieve and add it to our optimized ordering
optimizedOrdering.add(selected);
remainingSieveIndices.remove(selected);
}
logger.info("Final Sieve Ordering: " + StringUtils.join(sieveClassNames, ","));
logger.info("=============SIEVE OPTIMIZATION DONE ====================");
}
/**
* Extracts coreference clusters.
* This is the main API entry point for coreference resolution.
* Return a map from CorefChain ID to corresponding CorefChain.
* @throws Exception
*/
public Map<Integer, CorefChain> coref(Document document) throws Exception {
// Multi-pass sieve coreference resolution
for (int i = 0; i < sieves.length ; i++){
currentSieve = i;
DeterministicCorefSieve sieve = sieves[i];
// Do coreference resolution using this pass
coreference(document, sieve);
}
// post processing (e.g., removing singletons, appositions for conll)
if((!Constants.USE_GOLD_MENTIONS && doPostProcessing) || replicateCoNLL) postProcessing(document);
// coref system output: CorefChain
Map<Integer, CorefChain> result = Generics.newHashMap();
for(CorefCluster c : document.corefClusters.values()) {
result.put(c.clusterID, new CorefChain(c, document.positions));
}
return result;
}
public Map<Integer, edu.stanford.nlp.coref.data.CorefChain> corefReturnHybridOutput(Document document) throws Exception {
// Multi-pass sieve coreference resolution
for (int i = 0; i < sieves.length ; i++){
currentSieve = i;
DeterministicCorefSieve sieve = sieves[i];
// Do coreference resolution using this pass
coreference(document, sieve);
}
// post processing (e.g., removing singletons, appositions for conll)
if((!Constants.USE_GOLD_MENTIONS && doPostProcessing) || replicateCoNLL) postProcessing(document);
// coref system output: edu.stanford.nlp.hcoref.data.CorefChain
Map<Integer, edu.stanford.nlp.coref.data.CorefChain> result = Generics.newHashMap();
for(CorefCluster c : document.corefClusters.values()) {
// build mentionsMap and represents
Map<IntPair, Set<edu.stanford.nlp.coref.data.CorefChain.CorefMention>> mentionsMap = Generics.newHashMap();
IntPair keyPair = new IntPair(0,0);
mentionsMap.put(keyPair, new HashSet<>());
Mention represents = null;
edu.stanford.nlp.coref.data.CorefChain.CorefMention representsHybridVersion = null;
for (Mention mention : c.getCorefMentions()) {
// convert dcoref CorefMention to hcoref CorefMention
//IntPair mentionPosition = new IntPair(mention.sentNum, mention.headIndex);
IntTuple mentionPosition = document.positions.get(mention);
CorefMention dcorefMention = new CorefMention(mention, mentionPosition);
// tokens need the hcoref version of CorefClusterIdAnnotation
mention.headWord.set(edu.stanford.nlp.coref.CorefCoreAnnotations.CorefClusterIdAnnotation.class,
mention.corefClusterID);
// drop the dcoref version of CorefClusterIdAnnotation
mention.headWord.remove(CorefCoreAnnotations.CorefClusterIdAnnotation.class);
// make the hcoref mention
edu.stanford.nlp.coref.data.CorefChain.CorefMention hcorefMention =
new edu.stanford.nlp.coref.data.CorefChain.CorefMention(
edu.stanford.nlp.coref.data.Dictionaries.MentionType.valueOf(dcorefMention.mentionType.name()),
edu.stanford.nlp.coref.data.Dictionaries.Number.valueOf(dcorefMention.number.name()),
edu.stanford.nlp.coref.data.Dictionaries.Gender.valueOf(dcorefMention.gender.name()),
edu.stanford.nlp.coref.data.Dictionaries.Animacy.valueOf(dcorefMention.animacy.name()),
dcorefMention.startIndex,
dcorefMention.endIndex,
dcorefMention.headIndex,
dcorefMention.corefClusterID,
dcorefMention.mentionID,
dcorefMention.sentNum,
dcorefMention.position,
dcorefMention.mentionSpan);
mentionsMap.get(keyPair).add(hcorefMention);
if (mention.moreRepresentativeThan(represents)) {
represents = mention;
representsHybridVersion = hcorefMention;
}
}
edu.stanford.nlp.coref.data.CorefChain hybridCorefChain =
new edu.stanford.nlp.coref.data.CorefChain(c.clusterID, mentionsMap, representsHybridVersion);
result.put(c.clusterID, hybridCorefChain);
}
return result;
}
/**
* Do coreference resolution using one sieve pass.
*
* @param document An extracted document
* @throws Exception
*/
private void coreference(
Document document,
DeterministicCorefSieve sieve) throws Exception {
//Redwood.forceTrack("Coreference: sieve " + sieve.getClass().getSimpleName());
logger.finer("Coreference: sieve " + sieve.getClass().getSimpleName());
List<List<Mention>> orderedMentionsBySentence = document.getOrderedMentions();
Map<Integer, CorefCluster> corefClusters = document.corefClusters;
Set<Mention> roleSet = document.roleSet;
logger.finest("ROLE SET (Skip exact string match): ------------------");
for(Mention m : roleSet){
logger.finest("\t"+m.spanToString());
}
logger.finest("-------------------------------------------------------");
additionalCorrectLinksCount = 0;
additionalLinksCount = 0;
for (int sentI = 0; sentI < orderedMentionsBySentence.size(); sentI++) {
List<Mention> orderedMentions = orderedMentionsBySentence.get(sentI);
for (int mentionI = 0; mentionI < orderedMentions.size(); mentionI++) {
Mention m1 = orderedMentions.get(mentionI);
// check for skip: first mention only, discourse salience
if(sieve.skipThisMention(document, m1, corefClusters.get(m1.corefClusterID), dictionaries)) {
continue;
}
LOOP:
for (int sentJ = sentI; sentJ >= 0; sentJ--) {
List<Mention> l = sieve.getOrderedAntecedents(sentJ, sentI, orderedMentions, orderedMentionsBySentence, m1, mentionI, corefClusters, dictionaries);
if(maxSentDist != -1 && sentI - sentJ > maxSentDist) continue;
// Sort mentions by length whenever we have two mentions beginning at the same position and having the same head
for(int i = 0; i < l.size(); i++) {
for(int j = 0; j < l.size(); j++) {
if(l.get(i).headString.equals(l.get(j).headString) &&
l.get(i).startIndex == l.get(j).startIndex &&
l.get(i).sameSentence(l.get(j)) && j > i &&
l.get(i).spanToString().length() > l.get(j).spanToString().length()) {
logger.finest("FLIPPED: "+l.get(i).spanToString()+"("+i+"), "+l.get(j).spanToString()+"("+j+")");
l.set(j, l.set(i, l.get(j)));
}
}
}
for (Mention m2 : l) {
// m2 - antecedent of m1 l
// Skip singletons according to the singleton predictor
// (only for non-NE mentions)
// Recasens, de Marneffe, and Potts (NAACL 2013)
if (m1.isSingleton && m1.mentionType != MentionType.PROPER && m2.isSingleton && m2.mentionType != MentionType.PROPER) continue;
if (m1.corefClusterID == m2.corefClusterID) continue;
CorefCluster c1 = corefClusters.get(m1.corefClusterID);
CorefCluster c2 = corefClusters.get(m2.corefClusterID);
if (c2 == null) {
logger.warning("NO corefcluster id " + m2.corefClusterID);
}
assert(c1 != null);
assert(c2 != null);
if (sieve.useRoleSkip()) {
if (m1.isRoleAppositive(m2, dictionaries)) {
roleSet.add(m1);
} else if (m2.isRoleAppositive(m1, dictionaries)) {
roleSet.add(m2);
}
continue;
}
if (sieve.coreferent(document, c1, c2, m1, m2, dictionaries, roleSet, semantics)) {
// print logs for analysis
if (doScore()) {
printLogs(c1, c2, m1, m2, document, currentSieve);
}
int removeID = c1.clusterID;
CorefCluster.mergeClusters(c2, c1);
document.mergeIncompatibles(c2, c1);
document.mergeAcronymCache(c2, c1);
// logger.warning("Removing cluster " + removeID + ", merged with " + c2.getClusterID());
corefClusters.remove(removeID);
break LOOP;
}
}
} // End of "LOOP"
}
}
// scoring
if(doScore()){
scoreMUC.get(currentSieve).calculateScore(document);
scoreBcubed.get(currentSieve).calculateScore(document);
scorePairwise.get(currentSieve).calculateScore(document);
if(currentSieve==0) {
scoreSingleDoc = new ArrayList<>();
scoreSingleDoc.add(new ScorerPairwise());
scoreSingleDoc.get(currentSieve).calculateScore(document);
additionalCorrectLinksCount = (int) scoreSingleDoc.get(currentSieve).precisionNumSum;
additionalLinksCount = (int) scoreSingleDoc.get(currentSieve).precisionDenSum;
} else {
scoreSingleDoc.add(new ScorerPairwise());
scoreSingleDoc.get(currentSieve).calculateScore(document);
additionalCorrectLinksCount = (int) (scoreSingleDoc.get(currentSieve).precisionNumSum - scoreSingleDoc.get(currentSieve-1).precisionNumSum);
additionalLinksCount = (int) (scoreSingleDoc.get(currentSieve).precisionDenSum - scoreSingleDoc.get(currentSieve-1).precisionDenSum);
}
linksCountInPass.get(currentSieve).setFirst(linksCountInPass.get(currentSieve).first() + additionalCorrectLinksCount);
linksCountInPass.get(currentSieve).setSecond(linksCountInPass.get(currentSieve).second() + additionalLinksCount);
printSieveScore(document, sieve);
}
//Redwood.endTrack("Coreference: sieve " + sieve.getClass().getSimpleName());
}
/** Remove singletons, appositive, predicate nominatives, relative pronouns */
private static void postProcessing(Document document) {
Set<Mention> removeSet = Generics.newHashSet();
Set<Integer> removeClusterSet = Generics.newHashSet();
for(CorefCluster c : document.corefClusters.values()){
Set<Mention> removeMentions = Generics.newHashSet();
for(Mention m : c.getCorefMentions()) {
if(Constants.REMOVE_APPOSITION_PREDICATENOMINATIVES
&& ((m.appositions!=null && m.appositions.size() > 0)
|| (m.predicateNominatives!=null && m.predicateNominatives.size() > 0)
|| (m.relativePronouns!=null && m.relativePronouns.size() > 0))){
removeMentions.add(m);
removeSet.add(m);
m.corefClusterID = m.mentionID;
}
}
c.corefMentions.removeAll(removeMentions);
if(Constants.REMOVE_SINGLETONS && c.getCorefMentions().size()==1) {
removeClusterSet.add(c.clusterID);
}
}
for (int removeId : removeClusterSet){
document.corefClusters.remove(removeId);
}
for(Mention m : removeSet){
document.positions.remove(m);
}
}
public static LogisticClassifier<String, String> getSingletonPredictorFromSerializedFile(String serializedFile) {
try {
ObjectInputStream ois = IOUtils.readStreamFromString(serializedFile);
Object o = ois.readObject();
if (o instanceof LogisticClassifier<?, ?>) {
return (LogisticClassifier<String, String>) o;
}
throw new ClassCastException("Wanted SingletonPredictor, got " + o.getClass());
} catch (IOException e) {
throw new RuntimeIOException(e);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
/** Remove singleton clusters */
public static List<List<Mention>> filterMentionsWithSingletonClusters(Document document, List<List<Mention>> mentions)
{
List<List<Mention>> res = new ArrayList<>(mentions.size());
for (List<Mention> ml:mentions) {
List<Mention> filtered = new ArrayList<>();
for (Mention m:ml) {
CorefCluster cluster = document.corefClusters.get(m.corefClusterID);
if (cluster != null && cluster.getCorefMentions().size() > 1) {
filtered.add(m);
}
}
res.add(filtered);
}
return res;
}
public static void runConllEval(String conllMentionEvalScript,
String goldFile, String predictFile, String evalFile, String errFile) throws IOException {
ProcessBuilder process = new ProcessBuilder(conllMentionEvalScript, "all", goldFile, predictFile);
PrintWriter out = new PrintWriter(new FileOutputStream(evalFile));
PrintWriter err = new PrintWriter(new FileOutputStream(errFile));
SystemUtils.run(process, out, err);
out.close();
err.close();
}
public static String getConllEvalSummary(String conllMentionEvalScript,
String goldFile, String predictFile) throws IOException {
ProcessBuilder process = new ProcessBuilder(conllMentionEvalScript, "all", goldFile, predictFile, "none");
StringOutputStream errSos = new StringOutputStream();
StringOutputStream outSos = new StringOutputStream();
PrintWriter out = new PrintWriter(outSos);
PrintWriter err = new PrintWriter(errSos);
SystemUtils.run(process, out, err);
out.close();
err.close();
String summary = outSos.toString();
String errStr = errSos.toString();
if ( ! errStr.isEmpty()) {
summary += "\nERROR: " + errStr;
}
Pattern pattern = Pattern.compile("\\d+\\.\\d\\d\\d+");
DecimalFormat df = new DecimalFormat("#.##");
Matcher matcher = pattern.matcher(summary);
while(matcher.find()) {
String number = matcher.group();
summary = summary.replaceFirst(number, df.format(Double.parseDouble(number)));
}
return summary;
}
/** Print logs for error analysis */
public void printTopK(Logger logger, Document document, Semantics semantics) {
List<List<Mention>> orderedMentionsBySentence = document.getOrderedMentions();
Map<Integer, CorefCluster> corefClusters = document.corefClusters;
Map<Mention, IntTuple> positions = document.allPositions;
Map<Integer, Mention> golds = document.allGoldMentions;
logger.fine("=======ERROR ANALYSIS=========================================================");
// Temporary sieve for getting ordered antecedents
DeterministicCorefSieve tmpSieve = new ExactStringMatch();
for (int i = 0 ; i < orderedMentionsBySentence.size(); i++) {
List<Mention> orderedMentions = orderedMentionsBySentence.get(i);
for (int j = 0 ; j < orderedMentions.size(); j++) {
Mention m = orderedMentions.get(j);
logger.fine("=========Line: "+i+"\tmention: "+j+"=======================================================");
logger.fine(m.spanToString()+"\tmentionID: "+m.mentionID+"\tcorefClusterID: "+m.corefClusterID+"\tgoldCorefClusterID: "+m.goldCorefClusterID);
CorefCluster corefCluster = corefClusters.get(m.corefClusterID);
if (corefCluster != null) {
corefCluster.printCorefCluster(logger);
} else {
logger.finer("CANNOT find coref cluster for cluster " + m.corefClusterID);
}
logger.fine("-------------------------------------------------------");
boolean oneRecallErrorPrinted = false;
boolean onePrecisionErrorPrinted = false;
boolean alreadyChoose = false;
for (int sentJ = i; sentJ >= 0; sentJ--) {
List<Mention> l = tmpSieve.getOrderedAntecedents(sentJ, i, orderedMentions, orderedMentionsBySentence, m, j, corefClusters, dictionaries);
// Sort mentions by length whenever we have two mentions beginning at the same position and having the same head
for(int ii = 0; ii < l.size(); ii++) {
for(int jj = 0; jj < l.size(); jj++) {
if(l.get(ii).headString.equals(l.get(jj).headString) &&
l.get(ii).startIndex == l.get(jj).startIndex &&
l.get(ii).sameSentence(l.get(jj)) && jj > ii &&
l.get(ii).spanToString().length() > l.get(jj).spanToString().length()) {
logger.finest("FLIPPED: "+l.get(ii).spanToString()+"("+ii+"), "+l.get(jj).spanToString()+"("+jj+")");
l.set(jj, l.set(ii, l.get(jj)));
}
}
}
logger.finest("Candidates in sentence #"+sentJ+" for mention: "+m.spanToString());
for(int ii = 0; ii < l.size(); ii ++){
logger.finest("\tCandidate #"+ii+": "+l.get(ii).spanToString());
}
for (Mention antecedent : l) {
boolean chosen = (m.corefClusterID == antecedent.corefClusterID);
IntTuple src = new IntTuple(2);
src.set(0,i);
src.set(1,j);
IntTuple ant = positions.get(antecedent);
if(ant==null) continue;
//correct=(chosen==goldLinks.contains(new Pair<IntTuple, IntTuple>(src,ant)));
boolean coreferent = golds.containsKey(m.mentionID)
&& golds.containsKey(antecedent.mentionID)
&& (golds.get(m.mentionID).goldCorefClusterID == golds.get(antecedent.mentionID).goldCorefClusterID);
boolean correct = (chosen == coreferent);
String chosenness = chosen ? "Chosen" : "Not Chosen";
String correctness = correct ? "Correct" : "Incorrect";
logger.fine("\t" + correctness +"\t\t" + chosenness + "\t"+antecedent.spanToString());
CorefCluster mC = corefClusters.get(m.corefClusterID);
CorefCluster aC = corefClusters.get(antecedent.corefClusterID);
if(chosen && !correct && !onePrecisionErrorPrinted && !alreadyChoose) {
onePrecisionErrorPrinted = true;
printLinkWithContext(logger, "\nPRECISION ERROR ", src, ant, document, semantics);
logger.fine("END of PRECISION ERROR LOG");
}
if(!chosen && !correct && !oneRecallErrorPrinted && (!alreadyChoose || (alreadyChoose && onePrecisionErrorPrinted))) {
oneRecallErrorPrinted = true;
printLinkWithContext(logger, "\nRECALL ERROR ", src, ant, document, semantics);
logger.finer("cluster info: ");
if (mC != null) {
mC.printCorefCluster(logger);
} else {
logger.finer("CANNOT find coref cluster for cluster " + m.corefClusterID);
}
logger.finer("----------------------------------------------------------");
if (aC != null) {
aC.printCorefCluster(logger);
} else {
logger.finer("CANNOT find coref cluster for cluster " + m.corefClusterID);
}
logger.finer("");
logger.fine("END of RECALL ERROR LOG");
}
if(chosen) alreadyChoose = true;
}
}
logger.fine("\n");
}
}
logger.fine("===============================================================================");
}
public void printF1(boolean printF1First) {
scoreMUC.get(sieveClassNames.length - 1).printF1(logger, printF1First);
scoreBcubed.get(sieveClassNames.length - 1).printF1(logger, printF1First);
scorePairwise.get(sieveClassNames.length - 1).printF1(logger, printF1First);
}
private void printSieveScore(Document document, DeterministicCorefSieve sieve) {
logger.fine("===========================================");
logger.fine("pass"+currentSieve+": "+ sieve.flagsToString());
scoreMUC.get(currentSieve).printF1(logger);
scoreBcubed.get(currentSieve).printF1(logger);
scorePairwise.get(currentSieve).printF1(logger);
logger.fine("# of Clusters: "+document.corefClusters.size() + ",\t# of additional links: "+additionalLinksCount
+",\t# of additional correct links: "+additionalCorrectLinksCount
+",\tprecision of new links: "+1.0*additionalCorrectLinksCount/additionalLinksCount);
logger.fine("# of total additional links: "+linksCountInPass.get(currentSieve).second()
+",\t# of total additional correct links: "+linksCountInPass.get(currentSieve).first()
+",\taccumulated precision of this pass: "+1.0*linksCountInPass.get(currentSieve).first()/linksCountInPass.get(currentSieve).second());
logger.fine("--------------------------------------");
}
/** Print coref link info */
private static void printLink(Logger logger, String header, IntTuple src, IntTuple dst, List<List<Mention>> orderedMentionsBySentence) {
Mention srcMention = orderedMentionsBySentence.get(src.get(0)).get(src.get(1));
Mention dstMention = orderedMentionsBySentence.get(dst.get(0)).get(dst.get(1));
if(src.get(0)==dst.get(0)) {
logger.fine(header + ": ["+srcMention.spanToString()+"](id="+srcMention.mentionID
+") in sent #"+src.get(0)+" => ["+dstMention.spanToString()+"](id="+dstMention.mentionID+") in sent #"+dst.get(0) + " Same Sentence");
} else {
logger.fine(header + ": ["+srcMention.spanToString()+"](id="+srcMention.mentionID
+") in sent #"+src.get(0)+" => ["+dstMention.spanToString()+"](id="+dstMention.mentionID+") in sent #"+dst.get(0));
}
}
protected static void printList(Logger logger, String... args) {
StringBuilder sb = new StringBuilder();
for (String arg : args) {
sb.append(arg);
sb.append('\t');
}
logger.fine(sb.toString());
}
/** print a coref link information including context and parse tree */
private static void printLinkWithContext(Logger logger,
String header,
IntTuple src,
IntTuple dst,
Document document,
Semantics semantics
) {
List<List<Mention>> orderedMentionsBySentence = document.getOrderedMentions();
List<List<Mention>> goldOrderedMentionsBySentence = document.goldOrderedMentionsBySentence;
Mention srcMention = orderedMentionsBySentence.get(src.get(0)).get(src.get(1));
Mention dstMention = orderedMentionsBySentence.get(dst.get(0)).get(dst.get(1));
List<CoreLabel> srcSentence = srcMention.sentenceWords;
List<CoreLabel> dstSentence = dstMention.sentenceWords;
printLink(logger, header, src, dst, orderedMentionsBySentence);
printList(logger, "Mention:" + srcMention.spanToString(),
"Gender:" + srcMention.gender.toString(),
"Number:" + srcMention.number.toString(),
"Animacy:" + srcMention.animacy.toString(),
"Person:" + srcMention.person.toString(),
"NER:" + srcMention.nerString,
"Head:" + srcMention.headString,
"Type:" + srcMention.mentionType.toString(),
"utter: "+srcMention.headWord.get(CoreAnnotations.UtteranceAnnotation.class),
"speakerID: "+srcMention.headWord.get(CoreAnnotations.SpeakerAnnotation.class),
"twinless:" + srcMention.twinless);
logger.fine("Context:");
String p = "";
for(int i = 0; i < srcSentence.size(); i++) {
if (i == srcMention.startIndex) {
p += "[";
}
if (i == srcMention.endIndex) {
p += "]";
}
p += srcSentence.get(i).word() + " ";
}
logger.fine(p);
StringBuilder golds = new StringBuilder();
golds.append("Gold mentions in the sentence:\n");
Counter<Integer> mBegin = new ClassicCounter<>();
Counter<Integer> mEnd = new ClassicCounter<>();
for(Mention m : goldOrderedMentionsBySentence.get(src.get(0))){
mBegin.incrementCount(m.startIndex);
mEnd.incrementCount(m.endIndex);
}
List<CoreLabel> l = document.annotation.get(CoreAnnotations.SentencesAnnotation.class).get(src.get(0)).get(CoreAnnotations.TokensAnnotation.class);
for(int i = 0 ; i < l.size() ; i++){
for(int j = 0; j < mEnd.getCount(i); j++){
golds.append("]");
}
for(int j = 0; j < mBegin.getCount(i); j++){
golds.append("[");
}
golds.append(l.get(i).get(CoreAnnotations.TextAnnotation.class));
golds.append(" ");
}
logger.fine(golds.toString());
printList(logger, "\nAntecedent:" + dstMention.spanToString(),
"Gender:" + dstMention.gender.toString(),
"Number:" + dstMention.number.toString(),
"Animacy:" + dstMention.animacy.toString(),
"Person:" + dstMention.person.toString(),
"NER:" + dstMention.nerString,
"Head:" + dstMention.headString,
"Type:" + dstMention.mentionType.toString(),
"utter: "+dstMention.headWord.get(CoreAnnotations.UtteranceAnnotation.class),
"speakerID: "+dstMention.headWord.get(CoreAnnotations.SpeakerAnnotation.class),
"twinless:" + dstMention.twinless);
logger.fine("Context:");
p = "";
for(int i = 0; i < dstSentence.size(); i++) {
if (i == dstMention.startIndex) {
p += "[";
}
if (i == dstMention.endIndex) {
p += "]";
}
p += dstSentence.get(i).word() + " ";
}
logger.fine(p);
golds = new StringBuilder();
golds.append("Gold mentions in the sentence:\n");
mBegin = new ClassicCounter<>();
mEnd = new ClassicCounter<>();
for(Mention m : goldOrderedMentionsBySentence.get(dst.get(0))){
mBegin.incrementCount(m.startIndex);
mEnd.incrementCount(m.endIndex);
}
l = document.annotation.get(CoreAnnotations.SentencesAnnotation.class).get(dst.get(0)).get(CoreAnnotations.TokensAnnotation.class);
for(int i = 0 ; i < l.size() ; i++){
for(int j = 0; j < mEnd.getCount(i); j++){
golds.append("]");
}
for(int j = 0; j < mBegin.getCount(i); j++){
golds.append("[");
}
golds.append(l.get(i).get(CoreAnnotations.TextAnnotation.class));
golds.append(" ");
}
logger.fine(golds.toString());
logger.finer("\nMention:: --------------------------------------------------------");
try {
logger.finer(srcMention.dependency.toString());
} catch (Exception e){} //throw new RuntimeException(e);}
logger.finer("Parse:");
logger.finer(formatPennTree(srcMention.contextParseTree));
logger.finer("\nAntecedent:: -----------------------------------------------------");
try {
logger.finer(dstMention.dependency.toString());
} catch (Exception e){} //throw new RuntimeException(e);}
logger.finer("Parse:");
logger.finer(formatPennTree(dstMention.contextParseTree));
}
/** For printing tree in a better format */
private static String formatPennTree(Tree parseTree) {
String treeString = parseTree.pennString();
treeString = treeString.replaceAll("\\[TextAnnotation=", "");
treeString = treeString.replaceAll("(NamedEntityTag|Value|Index|PartOfSpeech)Annotation.+?\\)", ")");
treeString = treeString.replaceAll("\\[.+?\\]", "");
return treeString;
}
/** Print pass results */
private static void printLogs(CorefCluster c1, CorefCluster c2, Mention m1,
Mention m2, Document document, int sieveIndex) {
Map<Mention, IntTuple> positions = document.positions;
List<List<Mention>> orderedMentionsBySentence = document.getOrderedMentions();
List<Pair<IntTuple, IntTuple>> goldLinks = document.getGoldLinks();
IntTuple p1 = positions.get(m1);
assert(p1 != null);
IntTuple p2 = positions.get(m2);
assert(p2 != null);
int menDist = 0;
for (int i = p2.get(0) ; i<= p1.get(0) ; i++){
if(p1.get(0)==p2.get(0)) {
menDist = p1.get(1)-p2.get(1);
break;
}
if(i==p2.get(0)) {
menDist += orderedMentionsBySentence.get(p2.get(0)).size()-p2.get(1);
continue;
}
if(i==p1.get(0)) {
menDist += p1.get(1);
continue;
}
if(p2.get(0)<i && i < p1.get(0)) menDist += orderedMentionsBySentence.get(i).size();
}
String correct = (goldLinks.contains(new Pair<>(p1, p2)))? "\tCorrect" : "\tIncorrect";
logger.finest("\nsentence distance: "+(p1.get(0)-p2.get(0))+"\tmention distance: "+menDist + correct);
if(!goldLinks.contains(new Pair<>(p1, p2))){
logger.finer("-------Incorrect merge in pass"+sieveIndex+"::--------------------");
c1.printCorefCluster(logger);
logger.finer("--------------------------------------------");
c2.printCorefCluster(logger);
logger.finer("--------------------------------------------");
}
logger.finer("antecedent: "+m2.spanToString()+"("+m2.mentionID+")\tmention: "+m1.spanToString()+"("+m1.mentionID+")\tsentDistance: "+Math.abs(m1.sentNum-m2.sentNum)+"\t"+correct+" Pass"+sieveIndex+":");
}
private static void printDiscourseStructure(Document document) {
logger.finer("DISCOURSE STRUCTURE==============================");
logger.finer("doc type: "+document.docType);
int previousUtterIndex = -1;
String previousSpeaker = "";
StringBuilder sb = new StringBuilder();
for(CoreMap s : document.annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
for(CoreLabel l : s.get(CoreAnnotations.TokensAnnotation.class)) {
int utterIndex = l.get(CoreAnnotations.UtteranceAnnotation.class);
String speaker = l.get(CoreAnnotations.SpeakerAnnotation.class);
String word = l.get(CoreAnnotations.TextAnnotation.class);
if(previousUtterIndex!=utterIndex) {
try {
int previousSpeakerID = Integer.parseInt(previousSpeaker);
logger.finer("\n<utter>: "+previousUtterIndex + " <speaker>: "+document.allPredictedMentions.get(previousSpeakerID).spanToString());
} catch (Exception e) {
logger.finer("\n<utter>: "+previousUtterIndex + " <speaker>: "+previousSpeaker);
}
logger.finer(sb.toString());
sb.setLength(0);
previousUtterIndex = utterIndex;
previousSpeaker = speaker;
}
sb.append(" ").append(word);
}
sb.append("\n");
}
try {
int previousSpeakerID = Integer.parseInt(previousSpeaker);
logger.finer("\n<utter>: "+previousUtterIndex + " <speaker>: "+document.allPredictedMentions.get(previousSpeakerID).spanToString());
} catch (Exception e) {
logger.finer("\n<utter>: "+previousUtterIndex + " <speaker>: "+previousSpeaker);
}
logger.finer(sb.toString());
logger.finer("END OF DISCOURSE STRUCTURE==============================");
}
private static void printScoreSummary(String summary, Logger logger, boolean afterPostProcessing) {
String[] lines = summary.split("\n");
if(!afterPostProcessing) {
for(String line : lines) {
if(line.startsWith("Identification of Mentions")) {
logger.info(line);
return;
}
}
} else {
StringBuilder sb = new StringBuilder();
for(String line : lines) {
if(line.startsWith("METRIC")) sb.append(line);
if(!line.startsWith("Identification of Mentions") && line.contains("Recall")) {
sb.append(line).append("\n");
}
}
logger.info(sb.toString());
}
}
/** Print average F1 of MUC, B^3, CEAF_E */
private static void printFinalConllScore(String summary) {
Pattern f1 = Pattern.compile("Coreference:.*F1: (.*)%");
Matcher f1Matcher = f1.matcher(summary);
double[] F1s = new double[5];
int i = 0;
while (f1Matcher.find()) {
F1s[i++] = Double.parseDouble(f1Matcher.group(1));
}
double finalScore = (F1s[0]+F1s[1]+F1s[3])/3;
logger.info("Final conll score ((muc+bcub+ceafe)/3) = " + (new DecimalFormat("#.##")).format(finalScore));
}
private static double getFinalConllScore(String summary, String metricType, String scoreType) {
// metricType can be muc, bcub, ceafm, ceafe or combined
// Expects to match metricType muc, bcub, ceafm, ceafe
// Will not match the BLANC metrics (coref links, noncoref links, overall)
Pattern pattern = Pattern.compile("METRIC\\s+(.*):Coreference:.*" + scoreType + ":\\s*(\\([ 0-9./]*\\))?\\s*(\\d+(\\.\\d+)?)%");
Matcher matcher = pattern.matcher(summary);
double[] scores = new double[5];
String[] names = new String[5];
int i = 0;
while (matcher.find()) {
names[i] = matcher.group(1);
scores[i] = Double.parseDouble(matcher.group(3));
i++;
}
metricType = metricType.toLowerCase();
if ("combined".equals(metricType)) {
double finalScore = (scores[0]+scores[1]+scores[3])/3;
logger.info("Final conll score ((muc+bcub+ceafe)/3) " + scoreType + " = " + finalScore);
return finalScore;
} else {
if ("bcubed".equals(metricType)) {
metricType = "bcub";
}
for (i = 0; i < names.length; i++) {
if (names[i] != null && names[i].equals(metricType)) {
double finalScore = scores[i];
logger.info("Final conll score (" + metricType + ") " + scoreType + " = " + finalScore);
return finalScore;
}
}
throw new IllegalArgumentException("Invalid metricType:" + metricType);
}
}
/** Returns final selected score */
private double getFinalScore(String metricType, CorefScorer.SubScoreType subScoreType) {
metricType = metricType.toLowerCase();
int passIndex = sieveClassNames.length - 1;
String scoreDesc = metricType;
double finalScore;
switch (metricType) {
case "combined":
finalScore = (scoreMUC.get(passIndex).getScore(subScoreType)
+ scoreBcubed.get(passIndex).getScore(subScoreType)
+ scorePairwise.get(passIndex).getScore(subScoreType)) / 3;
scoreDesc = "(muc + bcub + pairwise)/3";
break;
case "muc":
finalScore = scoreMUC.get(passIndex).getScore(subScoreType);
break;
case "bcub":
case "bcubed":
finalScore = scoreBcubed.get(passIndex).getScore(subScoreType);
break;
case "pairwise":
finalScore = scorePairwise.get(passIndex).getScore(subScoreType);
break;
default:
throw new IllegalArgumentException("Invalid sub score type:" + subScoreType);
}
logger.info("Final score (" + scoreDesc + ") " + subScoreType + " = " + (new DecimalFormat("#.##")).format(finalScore));
return finalScore;
}
public static void printConllOutput(Document document, PrintWriter writer, boolean gold) {
printConllOutput(document, writer, gold, false);
}
public static void printConllOutput(Document document, PrintWriter writer, boolean gold, boolean filterSingletons) {
List<List<Mention>> orderedMentions;
if (gold) {
orderedMentions = document.goldOrderedMentionsBySentence;
} else {
orderedMentions = document.predictedOrderedMentionsBySentence;
}
if (filterSingletons) {
orderedMentions = filterMentionsWithSingletonClusters(document, orderedMentions);
}
printConllOutput(document, writer, orderedMentions, gold);
}
private static void printConllOutput(Document document, PrintWriter writer, List<List<Mention>> orderedMentions, boolean gold) {
Annotation anno = document.annotation;
List<List<String[]>> conllDocSentences = document.conllDoc.sentenceWordLists;
String docID = anno.get(CoreAnnotations.DocIDAnnotation.class);
StringBuilder sb = new StringBuilder();
sb.append("#begin document ").append(docID).append("\n");
List<CoreMap> sentences = anno.get(CoreAnnotations.SentencesAnnotation.class);
for(int sentNum = 0 ; sentNum < sentences.size() ; sentNum++){
List<CoreLabel> sentence = sentences.get(sentNum).get(CoreAnnotations.TokensAnnotation.class);
List<String[]> conllSentence = conllDocSentences.get(sentNum);
Map<Integer,Set<Mention>> mentionBeginOnly = Generics.newHashMap();
Map<Integer,Set<Mention>> mentionEndOnly = Generics.newHashMap();
Map<Integer,Set<Mention>> mentionBeginEnd = Generics.newHashMap();
for(int i=0 ; i<sentence.size(); i++){
mentionBeginOnly.put(i, new LinkedHashSet<>());
mentionEndOnly.put(i, new LinkedHashSet<>());
mentionBeginEnd.put(i, new LinkedHashSet<>());
}
for(Mention m : orderedMentions.get(sentNum)) {
if(m.startIndex==m.endIndex-1) {
mentionBeginEnd.get(m.startIndex).add(m);
} else {
mentionBeginOnly.get(m.startIndex).add(m);
mentionEndOnly.get(m.endIndex-1).add(m);
}
}
for(int i=0 ; i<sentence.size(); i++){
StringBuilder sb2 = new StringBuilder();
for(Mention m : mentionBeginOnly.get(i)){
if (sb2.length() > 0) {
sb2.append("|");
}
int corefClusterId = (gold)? m.goldCorefClusterID:m.corefClusterID;
sb2.append("(").append(corefClusterId);
}
for(Mention m : mentionBeginEnd.get(i)){
if (sb2.length() > 0) {
sb2.append("|");
}
int corefClusterId = (gold)? m.goldCorefClusterID:m.corefClusterID;
sb2.append("(").append(corefClusterId).append(")");
}
for(Mention m : mentionEndOnly.get(i)){
if (sb2.length() > 0) {
sb2.append("|");
}
int corefClusterId = (gold)? m.goldCorefClusterID:m.corefClusterID;
sb2.append(corefClusterId).append(")");
}
if(sb2.length() == 0) sb2.append("-");
String[] columns = conllSentence.get(i);
for(int j = 0 ; j < columns.length-1 ; j++){
String column = columns[j];
sb.append(column).append("\t");
}
sb.append(sb2).append("\n");
}
sb.append("\n");
}
sb.append("#end document").append("\n");
// sb.append("#end document ").append(docID).append("\n");
writer.print(sb.toString());
writer.flush();
}
/** Print raw document for analysis */
private static void printRawDoc(Document document, boolean gold) throws FileNotFoundException {
List<CoreMap> sentences = document.annotation.get(CoreAnnotations.SentencesAnnotation.class);
List<List<Mention>> allMentions;
if (gold) {
allMentions = document.goldOrderedMentionsBySentence;
} else {
allMentions = document.predictedOrderedMentionsBySentence;
}
// String filename = document.annotation.get()
StringBuilder doc = new StringBuilder();
int previousOffset = 0;
for(int i = 0 ; i<sentences.size(); i++) {
CoreMap sentence = sentences.get(i);
List<Mention> mentions = allMentions.get(i);
List<CoreLabel> t = sentence.get(CoreAnnotations.TokensAnnotation.class);
String[] tokens = new String[t.size()];
for(CoreLabel c : t) {
tokens[c.index()-1] = c.word();
}
if(previousOffset+2 < t.get(0).get(CoreAnnotations.CharacterOffsetBeginAnnotation.class)) {
doc.append("\n");
}
previousOffset = t.get(t.size()-1).get(CoreAnnotations.CharacterOffsetEndAnnotation.class);
Counter<Integer> startCounts = new ClassicCounter<>();
Counter<Integer> endCounts = new ClassicCounter<>();
Map<Integer, Set<Mention>> endMentions = Generics.newHashMap();
for (Mention m : mentions) {
startCounts.incrementCount(m.startIndex);
endCounts.incrementCount(m.endIndex);
if(!endMentions.containsKey(m.endIndex)) endMentions.put(m.endIndex, Generics.<Mention>newHashSet());
endMentions.get(m.endIndex).add(m);
}
for (int j = 0 ; j < tokens.length; j++){
if(endMentions.containsKey(j)) {
for(Mention m : endMentions.get(j)){
int corefChainId = (gold)? m.goldCorefClusterID: m.corefClusterID;
doc.append("]_").append(corefChainId);
}
}
for (int k = 0 ; k < startCounts.getCount(j) ; k++) {
if (doc.length() > 0 && doc.charAt(doc.length()-1) != '[') doc.append(" ");
doc.append("[");
}
if (doc.length() > 0 && doc.charAt(doc.length()-1)!='[') doc.append(" ");
doc.append(tokens[j]);
}
if(endMentions.containsKey(tokens.length)) {
for(Mention m : endMentions.get(tokens.length)){
int corefChainId = (gold)? m.goldCorefClusterID: m.corefClusterID;
doc.append("]_").append(corefChainId); //append("_").append(m.mentionID);
}
}
doc.append("\n");
}
logger.fine(document.annotation.get(CoreAnnotations.DocIDAnnotation.class));
if (gold) {
logger.fine("New DOC: (GOLD MENTIONS) ==================================================");
} else {
logger.fine("New DOC: (Predicted Mentions) ==================================================");
}
logger.fine(doc.toString());
}
public static List<Pair<IntTuple, IntTuple>> getLinks(
Map<Integer, CorefChain> result) {
List<Pair<IntTuple, IntTuple>> links = new ArrayList<>();
CorefChain.CorefMentionComparator comparator = new CorefChain.CorefMentionComparator();
for (CorefChain c : result.values()) {
List<CorefMention> s = c.getMentionsInTextualOrder();
for (CorefMention m1 : s) {
for (CorefMention m2 : s) {
if (comparator.compare(m1, m2)==1) {
links.add(new Pair<>(m1.position, m2.position));
}
}
}
}
return links;
}
public static void debugPrintMentions(PrintStream out, String tag, List<List<Mention>> mentions) {
for(int i = 0; i < mentions.size(); i ++){
out.println(tag + " SENTENCE " + i);
for(int j = 0; j < mentions.get(i).size(); j ++){
Mention m = mentions.get(i).get(j);
String ms = "(" + m.mentionID + "," + m.originalRef + "," + m.corefClusterID
+ ",[" + m.startIndex + "," + m.endIndex +"]" + ") ";
out.print(ms);
}
out.println();
}
}
public static boolean checkClusters(Logger logger, String tag, Document document) {
List<List<Mention>> mentions = document.getOrderedMentions();
boolean clustersOk = true;
for (List<Mention> mentionCluster : mentions) {
for (Mention m : mentionCluster) {
String ms = "(" + m.mentionID + "," + m.originalRef + "," + m.corefClusterID
+ ",[" + m.startIndex + "," + m.endIndex + "]" + ") ";
CorefCluster cluster = document.corefClusters.get(m.corefClusterID);
if (cluster == null) {
logger.warning(tag + ": Cluster not found for mention: " + ms);
clustersOk = false;
} else if (!cluster.getCorefMentions().contains(m)) {
logger.warning(tag + ": Cluster does not contain mention: " + ms);
clustersOk = false;
}
}
}
return clustersOk;
}
}