/*************************************************************************
* *
* This file is part of the 20n/act project. *
* 20n/act enables DNA prediction for synthetic biology/bioengineering. *
* Copyright (C) 2017 20n Labs, Inc. *
* *
* Please direct all queries to act@20n.com. *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
*************************************************************************/
package com.twentyn.patentTextProcessor;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.twentyn.patentExtractor.PatentCorpusReader;
import com.twentyn.patentExtractor.PatentDocument;
import com.twentyn.patentExtractor.PatentProcessor;
import edu.emory.clir.clearnlp.component.utils.NLPUtils;
import edu.emory.clir.clearnlp.tokenization.AbstractTokenizer;
import edu.emory.clir.clearnlp.util.lang.TLanguage;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.input.ReaderInputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPathExpressionException;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
public class WordCountProcessor implements PatentProcessor {
public static final Logger LOGGER = Logger.getLogger(PatentProcessor.class);
public static class PatentDocumentWordCount {
@JsonProperty("patent_file_path")
String patentFilePath;
@JsonProperty("word_count")
Map<String, Integer> wordCount;
public PatentDocumentWordCount(String patentFilePath, Map<String, Integer> wordCount) {
this.patentFilePath = patentFilePath;
this.wordCount = wordCount;
}
public String getPatentFilePath() {
return patentFilePath;
}
public Map<String, Integer> getWordCount() {
return wordCount;
}
}
ObjectMapper objectMapper = new ObjectMapper();
{
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
objectMapper.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
}
private AbstractTokenizer tokenizer;
private Set<String> stopWords = new HashSet<>();
public WordCountProcessor() {
this.tokenizer = NLPUtils.getTokenizer(TLanguage.ENGLISH);
}
public void readStopWords(File path) throws IOException{
try (BufferedReader r = new BufferedReader(new FileReader(path))) {
String line = null;
while ((line = r.readLine()) != null) {
this.stopWords.add(line);
}
}
}
public static final Pattern EXCLUDE_NUMBERS = Pattern.compile("^[+\\-]?[0-9\\.]+$");
public void processPatentText(File patentFile, Reader patentTextReader, int patentTextLength)
throws IOException, ParserConfigurationException, SAXException,
TransformerConfigurationException, TransformerException, XPathExpressionException {
PatentDocument doc = PatentDocument.patentDocumentFromXMLStream(new ReaderInputStream(patentTextReader));
if (doc == null) {
LOGGER.warn(String.format("Got null patent document object for patent at %s", patentFile.getAbsolutePath()));
return;
}
// Put all the text back together into one big string. Sigh.
List<String> textFields = new ArrayList<String>(1 + doc.getClaimsText().size() + doc.getTextContent().size());
textFields.add(doc.getTitle());
textFields.addAll(doc.getClaimsText());
textFields.addAll(doc.getTextContent());
String joinedText = StringUtils.join(textFields, "\n");
// TODO: parallelize this. Because come on, it's the nineties and Map Reduce is almost a thousand years old.
Map<String, Integer> wordCount = new HashMap<>();
List<String> words = this.tokenizer.tokenize(joinedText);
for (String word : words) {
if (EXCLUDE_NUMBERS.matcher(word).matches()) {
continue; // Ignore words that are just numbers (with decimals)
}
word = word.toLowerCase();
if (stopWords.contains(word)) {
continue; // Ignore stop words;
}
Integer count = wordCount.get(word);
if (count == null) {
count = 1;
}
wordCount.put(word, count + 1);
}
PatentDocumentWordCount pdwc = new PatentDocumentWordCount(patentFile.getAbsolutePath(), wordCount);
System.out.println(objectMapper.writeValueAsString(pdwc));
}
public static void main(String[] args) throws Exception {
System.out.println("Starting up...");
System.out.flush();
Options opts = new Options();
opts.addOption(Option.builder("i").
longOpt("input").hasArg().required().desc("Input file or directory to score").build());
opts.addOption(Option.builder("h").longOpt("help").desc("Print this help message and exit").build());
opts.addOption(Option.builder("v").longOpt("verbose").desc("Print verbose log output").build());
HelpFormatter helpFormatter = new HelpFormatter();
CommandLineParser cmdLineParser = new DefaultParser();
CommandLine cmdLine = null;
try {
cmdLine = cmdLineParser.parse(opts, args);
} catch (ParseException e) {
System.out.println("Caught exception when parsing command line: " + e.getMessage());
helpFormatter.printHelp("WordCountProcessor", opts);
System.exit(1);
}
if (cmdLine.hasOption("help")) {
helpFormatter.printHelp("DocumentIndexer", opts);
System.exit(0);
}
String inputFileOrDir = cmdLine.getOptionValue("input");
File splitFileOrDir = new File(inputFileOrDir);
if (!(splitFileOrDir.exists())) {
LOGGER.error("Unable to find directory at " + inputFileOrDir);
System.exit(1);
}
WordCountProcessor wcp = new WordCountProcessor();
PatentCorpusReader corpusReader = new PatentCorpusReader(wcp, splitFileOrDir);
corpusReader.readPatentCorpus();
}
}