AnalyzerUtil.java

package org.apache.lucene.wordnet;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.io.PrintStream;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.regex.Pattern;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;

/**
 * Various fulltext analysis utilities avoiding redundant code in several
 * classes.
 *
 */
public class AnalyzerUtil {
  
  private AnalyzerUtil() {};

  /**
   * Returns a simple analyzer wrapper that logs all tokens produced by the
   * underlying child analyzer to the given log stream (typically System.err);
   * Otherwise behaves exactly like the child analyzer, delivering the very
   * same tokens; useful for debugging purposes on custom indexing and/or
   * querying.
   * 
   * @param child
   *            the underlying child analyzer
   * @param log
   *            the print stream to log to (typically System.err)
   * @param logName
   *            a name for this logger (typically "log" or similar)
   * @return a logging analyzer
   */
  public static Analyzer getLoggingAnalyzer(final Analyzer child, 
      final PrintStream log, final String logName) {
    
    if (child == null) 
      throw new IllegalArgumentException("child analyzer must not be null");
    if (log == null) 
      throw new IllegalArgumentException("logStream must not be null");

    return new Analyzer() {
      @Override
      public TokenStream tokenStream(final String fieldName, Reader reader) {
        return new TokenFilter(child.tokenStream(fieldName, reader)) {
          private int position = -1;
          private TermAttribute termAtt = addAttribute(TermAttribute.class);
          private PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
          private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
          private TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
         
          @Override
          public boolean incrementToken() throws IOException {
            boolean hasNext = input.incrementToken();
            log.println(toString(hasNext));
            return hasNext;
          }
          
          private String toString(boolean hasNext) {
            if (!hasNext) return "[" + logName + ":EOS:" + fieldName + "]\n";
            
            position += posIncrAtt.getPositionIncrement();
            return "[" + logName + ":" + position + ":" + fieldName + ":"
                + termAtt.term() + ":" + offsetAtt.startOffset()
                + "-" + offsetAtt.endOffset() + ":" + typeAtt.type()
                + "]";
          }         
        };
      }
    };
  }
  
  
  /**
   * Returns an analyzer wrapper that returns at most the first
   * <code>maxTokens</code> tokens from the underlying child analyzer,
   * ignoring all remaining tokens.
   * 
   * @param child
   *            the underlying child analyzer
   * @param maxTokens
   *            the maximum number of tokens to return from the underlying
   *            analyzer (a value of Integer.MAX_VALUE indicates unlimited)
   * @return an analyzer wrapper
   */
  public static Analyzer getMaxTokenAnalyzer(
      final Analyzer child, final int maxTokens) {
    
    if (child == null) 
      throw new IllegalArgumentException("child analyzer must not be null");
    if (maxTokens < 0) 
      throw new IllegalArgumentException("maxTokens must not be negative");
    if (maxTokens == Integer.MAX_VALUE) 
      return child; // no need to wrap
  
    return new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(child.tokenStream(fieldName, reader)) {
          private int todo = maxTokens;
          
          @Override
          public boolean incrementToken() throws IOException {
            return --todo >= 0 ? input.incrementToken() : false;
          }
        };
      }
    };
  }
  
  
  /**
   * Returns an English stemming analyzer that stems tokens from the
   * underlying child analyzer according to the Porter stemming algorithm. The
   * child analyzer must deliver tokens in lower case for the stemmer to work
   * properly.
   * <p>
   * Background: Stemming reduces token terms to their linguistic root form
   * e.g. reduces "fishing" and "fishes" to "fish", "family" and "families" to
   * "famili", as well as "complete" and "completion" to "complet". Note that
   * the root form is not necessarily a meaningful word in itself, and that
   * this is not a bug but rather a feature, if you lean back and think about
   * fuzzy word matching for a bit.
   * <p>
   * See the Lucene contrib packages for stemmers (and stop words) for German,
   * Russian and many more languages.
   * 
   * @param child
   *            the underlying child analyzer
   * @return an analyzer wrapper
   */
  public static Analyzer getPorterStemmerAnalyzer(final Analyzer child) {
    
    if (child == null) 
      throw new IllegalArgumentException("child analyzer must not be null");
  
    return new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new PorterStemFilter(
            child.tokenStream(fieldName, reader));
//        /* PorterStemFilter and SnowballFilter have the same behaviour, 
//        but PorterStemFilter is much faster. */
//        return new org.apache.lucene.analysis.snowball.SnowballFilter(
//            child.tokenStream(fieldName, reader), "English");
      }
    };
  }
  
  
  /**
   * Returns an analyzer wrapper that wraps the underlying child analyzer's
   * token stream into a {@link SynonymTokenFilter}.
   * 
   * @param child
   *            the underlying child analyzer
   * @param synonyms
   *            the map used to extract synonyms for terms
   * @param maxSynonyms
   *            the maximum number of synonym tokens to return per underlying
   *            token word (a value of Integer.MAX_VALUE indicates unlimited)
   * @return a new analyzer
   */
  public static Analyzer getSynonymAnalyzer(final Analyzer child, 
      final SynonymMap synonyms, final int maxSynonyms) {
    
    if (child == null) 
      throw new IllegalArgumentException("child analyzer must not be null");
    if (synonyms == null)
      throw new IllegalArgumentException("synonyms must not be null");
    if (maxSynonyms < 0) 
      throw new IllegalArgumentException("maxSynonyms must not be negative");
    if (maxSynonyms == 0)
      return child; // no need to wrap
  
    return new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new SynonymTokenFilter(
          child.tokenStream(fieldName, reader), synonyms, maxSynonyms);
      }
    };
  }

  
  /**
   * Returns an analyzer wrapper that caches all tokens generated by the underlying child analyzer's
   * token streams, and delivers those cached tokens on subsequent calls to 
   * <code>tokenStream(String fieldName, Reader reader)</code> 
   * if the fieldName has been seen before, altogether ignoring the Reader parameter on cache lookup.
   * <p>
   * If Analyzer / TokenFilter chains are expensive in terms of I/O or CPU, such caching can 
   * help improve performance if the same document is added to multiple Lucene indexes, 
   * because the text analysis phase need not be performed more than once.
   * <p>
   * Caveats: 
   * <ul>
   * <li>Caching the tokens of large Lucene documents can lead to out of memory exceptions.</li> 
   * <li>The Token instances delivered by the underlying child analyzer must be immutable.</li>
   * <li>The same caching analyzer instance must not be used for more than one document
   * because the cache is not keyed on the Reader parameter.</li>
   * </ul>
   * 
   * @param child
   *            the underlying child analyzer
   * @return a new analyzer
   */
  public static Analyzer getTokenCachingAnalyzer(final Analyzer child) {

    if (child == null)
      throw new IllegalArgumentException("child analyzer must not be null");

    return new Analyzer() {

      private final HashMap<String,ArrayList<AttributeSource.State>> cache = new HashMap<String,ArrayList<AttributeSource.State>>();

      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        final ArrayList<AttributeSource.State> tokens = cache.get(fieldName);
        if (tokens == null) { // not yet cached
          final ArrayList<AttributeSource.State> tokens2 = new ArrayList<AttributeSource.State>();
          TokenStream tokenStream = new TokenFilter(child.tokenStream(fieldName, reader)) {

            @Override
            public boolean incrementToken() throws IOException {
              boolean hasNext = input.incrementToken();
              if (hasNext) tokens2.add(captureState());
              return hasNext;
            }
          };
          
          cache.put(fieldName, tokens2);
          return tokenStream;
        } else { // already cached
          return new TokenStream() {

            private Iterator<AttributeSource.State> iter = tokens.iterator();

            @Override
            public boolean incrementToken() {
              if (!iter.hasNext()) return false;
              restoreState(iter.next());
              return true;
            }
          };
        }
      }
    };
  }
      
  
  /**
   * Returns (frequency:term) pairs for the top N distinct terms (aka words),
   * sorted descending by frequency (and ascending by term, if tied).
   * <p>
   * Example XQuery:
   * <pre>
   * declare namespace util = "java:org.apache.lucene.index.memory.AnalyzerUtil";
   * declare namespace analyzer = "java:org.apache.lucene.index.memory.PatternAnalyzer";
   * 
   * for $pair in util:get-most-frequent-terms(
   *    analyzer:EXTENDED_ANALYZER(), doc("samples/shakespeare/othello.xml"), 10)
   * return <word word="{substring-after($pair, ':')}" frequency="{substring-before($pair, ':')}"/>
   * </pre>
   * 
   * @param analyzer
   *            the analyzer to use for splitting text into terms (aka words)
   * @param text
   *            the text to analyze
   * @param limit
   *            the maximum number of pairs to return; zero indicates 
   *            "as many as possible".
   * @return an array of (frequency:term) pairs in the form of (freq0:term0,
   *         freq1:term1, ..., freqN:termN). Each pair is a single string
   *         separated by a ':' delimiter.
   */
  public static String[] getMostFrequentTerms(Analyzer analyzer, String text, int limit) {
    if (analyzer == null) 
      throw new IllegalArgumentException("analyzer must not be null");
    if (text == null) 
      throw new IllegalArgumentException("text must not be null");
    if (limit <= 0) limit = Integer.MAX_VALUE;
    
    // compute frequencies of distinct terms
    HashMap<String,MutableInteger> map = new HashMap<String,MutableInteger>();
    TokenStream stream = analyzer.tokenStream("", new StringReader(text));
    TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
    try {
      while (stream.incrementToken()) {
        MutableInteger freq = map.get(termAtt.term());
        if (freq == null) {
          freq = new MutableInteger(1);
          map.put(termAtt.term(), freq);
        } else {
          freq.setValue(freq.intValue() + 1);
        }
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    } finally {
      try {
        stream.close();
      } catch (IOException e2) {
        throw new RuntimeException(e2);
      }
    }
    
    // sort by frequency, text
    Map.Entry<String,MutableInteger>[] entries = new Map.Entry[map.size()];
    map.entrySet().toArray(entries);
    Arrays.sort(entries, new Comparator<Map.Entry<String,MutableInteger>>() {
      public int compare(Map.Entry<String,MutableInteger> e1, Map.Entry<String,MutableInteger> e2) {
        int f1 = e1.getValue().intValue();
        int f2 = e2.getValue().intValue();
        if (f2 - f1 != 0) return f2 - f1;
        String s1 = e1.getKey();
        String s2 = e2.getKey();
        return s1.compareTo(s2);
      }
    });
    
    // return top N entries
    int size = Math.min(limit, entries.length);
    String[] pairs = new String[size];
    for (int i=0; i < size; i++) {
      pairs[i] = entries[i].getValue() + ":" + entries[i].getKey();
    }
    return pairs;
  }
  
  private static final class MutableInteger {
    private int value;
    public MutableInteger(int value) { this.value = value; }
    public int intValue() { return value; }
    public void setValue(int value) { this.value = value; }
    @Override
    public String toString() { return String.valueOf(value); }
  };
  
  
  
  // TODO: could use a more general i18n approach ala http://icu.sourceforge.net/docs/papers/text_boundary_analysis_in_java/
  /** (Line terminator followed by zero or more whitespace) two or more times */
  private static final Pattern PARAGRAPHS = Pattern.compile("([\\r\\n\\u0085\\u2028\\u2029][ \\t\\x0B\\f]*){2,}");
  
  /**
   * Returns at most the first N paragraphs of the given text. Delimiting
   * characters are excluded from the results. Each returned paragraph is
   * whitespace-trimmed via String.trim(), potentially an empty string.
   * 
   * @param text
   *            the text to tokenize into paragraphs
   * @param limit
   *            the maximum number of paragraphs to return; zero indicates "as
   *            many as possible".
   * @return the first N paragraphs
   */
  public static String[] getParagraphs(String text, int limit) {
    return tokenize(PARAGRAPHS, text, limit);
  }
    
  private static String[] tokenize(Pattern pattern, String text, int limit) {
    String[] tokens = pattern.split(text, limit);
    for (int i=tokens.length; --i >= 0; ) tokens[i] = tokens[i].trim();
    return tokens;
  }
  
  
  // TODO: don't split on floating point numbers, e.g. 3.1415 (digit before or after '.')
  /** Divides text into sentences; Includes inverted spanish exclamation and question mark */
  private static final Pattern SENTENCES  = Pattern.compile("[!\\.\\?\\xA1\\xBF]+");

  /**
   * Returns at most the first N sentences of the given text. Delimiting
   * characters are excluded from the results. Each returned sentence is
   * whitespace-trimmed via String.trim(), potentially an empty string.
   * 
   * @param text
   *            the text to tokenize into sentences
   * @param limit
   *            the maximum number of sentences to return; zero indicates "as
   *            many as possible".
   * @return the first N sentences
   */
  public static String[] getSentences(String text, int limit) {
//    return tokenize(SENTENCES, text, limit); // equivalent but slower
    int len = text.length();
    if (len == 0) return new String[] { text };
    if (limit <= 0) limit = Integer.MAX_VALUE;
    
    // average sentence length heuristic
    String[] tokens = new String[Math.min(limit, 1 + len/40)];
    int size = 0;
    int i = 0;
    
    while (i < len && size < limit) {
      
      // scan to end of current sentence
      int start = i;
      while (i < len && !isSentenceSeparator(text.charAt(i))) i++;
      
      if (size == tokens.length) { // grow array
        String[] tmp = new String[tokens.length << 1];
        System.arraycopy(tokens, 0, tmp, 0, size);
        tokens = tmp;
      }
      // add sentence (potentially empty)
      tokens[size++] = text.substring(start, i).trim();

      // scan to beginning of next sentence
      while (i < len && isSentenceSeparator(text.charAt(i))) i++;
    }
    
    if (size == tokens.length) return tokens;
    String[] results = new String[size];
    System.arraycopy(tokens, 0, results, 0, size);
    return results;
  }

  private static boolean isSentenceSeparator(char c) {
    // regex [!\\.\\?\\xA1\\xBF]
    switch (c) {
      case '!': return true;
      case '.': return true;
      case '?': return true;
      case 0xA1: return true; // spanish inverted exclamation mark
      case 0xBF: return true; // spanish inverted question mark
      default: return false;
    }   
  }
  
}