package edu.stanford.nlp.ling.tokensregex.demo;

import edu.stanford.nlp.io.IOUtils;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.util.CoreMap;

import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
import java.util.Properties;

/**
 * Demo illustrating how to use TokensRegexAnnotator for tweaks to tokenization
 */
public class TokensRegexRetokenizeDemo {

  public static void runPipeline(StanfordCoreNLP pipeline, String text, PrintWriter out) {
    Annotation annotation = new Annotation(text);
    pipeline.annotate(annotation);

    // An Annotation is a Map and you can get and use the various analyses individually.
    out.println();
    // The toString() method on an Annotation just prints the text of the Annotation
    // But you can see what is in it with other methods like toShorterString()
    out.println("The top level annotation");
    out.println(annotation.toShorterString());
    List<CoreMap> sentences = annotation.get(CoreAnnotations.SentencesAnnotation.class);

    for (CoreMap sentence : sentences) {
      // Print out token annotations
      for (CoreLabel token:sentence.get(CoreAnnotations.TokensAnnotation.class)) {
        // Print out words, lemma, ne, and normalized ne
        String word = token.get(CoreAnnotations.TextAnnotation.class);
        String lemma = token.get(CoreAnnotations.LemmaAnnotation.class);
        String pos = token.get(CoreAnnotations.PartOfSpeechAnnotation.class);
        String ne = token.get(CoreAnnotations.NamedEntityTagAnnotation.class);
        String normalized = token.get(CoreAnnotations.NormalizedNamedEntityTagAnnotation.class);
        out.println("token: " + "word="+word + ", lemma="+lemma + ", pos=" + pos + ", ne=" + ne + ", normalized=" + normalized);
      }
    }
    out.flush();
  }

  public static void main(String[] args) throws IOException {
    PrintWriter out;

    String rules;
    if (args.length > 0) {
      rules = args[0];
    } else {
      rules = "edu/stanford/nlp/ling/tokensregex/demo/rules/retokenize.rules.txt";
    }
    if (args.length > 2) {
      out = new PrintWriter(args[2]);
    } else {
      out = new PrintWriter(System.out);
    }

    String text;
    if (args.length > 1) {
      text = IOUtils.slurpFileNoExceptions(args[1]);
    } else {
      text = "Do we tokenize on hyphens? one-two-three-four.  How about dates? 03-16-2015.";
    }

    Properties propertiesDefaultTokenize = new Properties();
    propertiesDefaultTokenize.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner");
    StanfordCoreNLP pipelineDefaultRetokenize = new StanfordCoreNLP();
    out.println("Default tokenization: ");
    runPipeline(pipelineDefaultRetokenize, text, out);

    Properties properties = new Properties();
    properties.setProperty("annotators", "tokenize,retokenize,ssplit,pos,lemma,ner");
    properties.setProperty("customAnnotatorClass.retokenize", "edu.stanford.nlp.pipeline.TokensRegexAnnotator");
    properties.setProperty("retokenize.rules", rules);
    StanfordCoreNLP pipelineWithRetokenize = new StanfordCoreNLP(properties);
    out.println();
    out.println("Always tokenize hyphens: ");
    runPipeline(pipelineWithRetokenize, text, out);
  }

}