ConfusionSetOccurrenceLookup.java

/* LanguageTool, a natural language style checker
 * Copyright (C) 2015 Daniel Naber (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.bigdata;

import org.languagetool.languagemodel.LuceneLanguageModel;

import java.io.File;
import java.io.IOException;
import java.util.*;

/**
 * Take simple confusion set file (one set per line, separated by semicolons) and print out
 * occurrence information for all items.
 * @since 3.1
 */
final class ConfusionSetOccurrenceLookup {

  private ConfusionSetOccurrenceLookup() {
  }

  public static void main(String[] args) throws IOException {
    if (args.length != 2) {
      System.out.println("Usage: " + ConfusionSetOccurrenceLookup.class.getName() + " <confusion-file> <ngram-data-dir>");
      System.exit(1);
    }
    try (Scanner sc = new Scanner(new File(args[0]));
         LuceneLanguageModel lm = new LuceneLanguageModel(new File(args[1]))
    ) {
      while (sc.hasNextLine()) {
        String line = sc.nextLine();
        String[] words = line.split(";\\s*");
        long total = 0;
        List<Long> counts = new ArrayList<>();
        StringBuilder sb = new StringBuilder();
        for (String word : words) {
          long count = lm.getCount(word);
          total += count;
          sb.append(word).append(":").append(count).append(" ");
          counts.add(count);
        }
        float factor = (float)Collections.max(counts) / Collections.min(counts);
        System.out.printf(Locale.ENGLISH, total + " " + line + "    " + sb.toString().trim() + " factor:%.1f\n", factor);
      }
    }
  }
  
}