/**
Copyright (C) SYSTAP, LLC DBA Blazegraph 2006-2016. All rights reserved.
Contact:
SYSTAP, LLC DBA Blazegraph
2501 Calvert ST NW #106
Washington, DC 20008
licenses@blazegraph.com
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
/*
* Created on May 8, 2014 by Jeremy J. Carroll, Syapse Inc.
*/
package com.bigdata.search;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.CharBuffer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* An analyzer intended for the term-completion use case; particularly
* for technical vocabularies and concept schemes.
*
* <p>
* This analyzer generates several index terms for each word in the input.
* These are intended to match short sequences (e.g. three or more) characters
* of user-input, to then give the user a drop-down list of matching terms.
* <p>
* This can be set up to address issues like matching <q>half-time</q> when the user types
* <q>tim</q> or if the user types <q>halft</q> (treating the hyphen as a soft hyphen); or
* to match <q>TermCompletionAnalyzer</q> when the user types <q>Ana</q>
* <p>
* In contrast, the Lucene Analyzers are mainly geared around the free text search use
* case.
* <p>
* The intended use cases will typical involve a prefix query of the form:
* <pre>
* ?t bds:search "prefix*" .
* </pre>
* to find all literals in the selected graphs, which are indexed by a term starting in <q>prefix</q>,
* so the problem this class addresses is finding the appropriate index terms to allow
* matching, at sensible points, mid-way through words (such as at hyphens).
* <p>
* To get maximum effectiveness it maybe best to use private language subtags (see RFC 5647),
* e.g. <code>"x-term"</code>
* which are mapped to this class by {@link ConfigurableAnalyzerFactory} for
* the data being loaded into the store, and linked to some very simple process
* like {@link KeywordAnalyzer} for queries which are tagged with a different language tag
* that is only used for <code>bds:search</code>, e.g. <code>"x-query"</code>.
* The above prefix query then becomes:
* <pre>
* ?t bds:search "prefix*"@x-query .
* </pre>
*
*
*
* @author jeremycarroll
*
*/
public class TermCompletionAnalyzer extends Analyzer {
private final Pattern wordBoundary;
private final Pattern subWordBoundary;
private final Pattern discard;
private final boolean alwaysDiscard;
/**
* Divide the input into words and short tokens
* as with {@link #TermCompletionAnalyzer(Pattern, Pattern)}.
* Each term is generated, and then an additional term
* is generated with softHypens (defined by the pattern),
* removed. If the alwaysRemoveSoftHypens flag is true,
* then the first term (before the removal) is suppressed.
*
* @param wordBoundary The definition of space (e.g. " ")
* @param subWordBoundary Also index after matches to this (e.g. "-")
* @param softHyphens Discard these characters from matches
* @param alwaysRemoveSoftHypens If false the discard step is optional.
*/
public TermCompletionAnalyzer(Pattern wordBoundary,
Pattern subWordBoundary,
Pattern softHyphens,
boolean alwaysRemoveSoftHypens) {
this.wordBoundary = wordBoundary;
this.subWordBoundary = subWordBoundary;
if (softHyphens != null) {
discard = softHyphens;
alwaysDiscard = alwaysRemoveSoftHypens;
} else {
discard = Pattern.compile("(?!)"); // never matches
alwaysDiscard = true;
}
}
/**
* Divide the input into words, separated by the wordBoundary,
* and return a token for each whole word, and then
* generate further tokens for each word by removing prefixes
* up to and including each successive match of
* subWordBoundary
* @param wordBoundary
* @param subWordBoundary
*/
public TermCompletionAnalyzer(Pattern wordBoundary,
Pattern subWordBoundary) {
this(wordBoundary, subWordBoundary, null, true);
}
/**
* This classes has three processes going on
* all driven from the {@link #increment()} method.
*
* One process is that of iterating over the words in the input:
* - the words are identified in the constructor, and the iteration
* is performed by {@link #nextWord()}
*
* - the subword boundaries are identified in {@link #next()}
* We then set up {@link #found} to contain the most
* recently found subword.
*
* - the soft hyphen discarding is processed in {@link #maybeDiscardHyphens()}
*
* - if we are not {@link #alwaysDiscard}ing then {@link #afterDiscard}
* can be set to null to return the non-discarded version on the next cycle.
*
*/
private class TermCompletionTokenStream extends TokenStream {
final String[] words;
final CharTermAttribute termAtt;
char currentWord[] = new char[]{};
Matcher softMatcher;
int currentWordIx = -1;
int charPos = 0;
private String afterDiscard;
private CharBuffer found;
public TermCompletionTokenStream(final Reader reader) {
termAtt = addAttribute(CharTermAttribute.class);
words = wordBoundary.split(getStringReaderContents(reader));
}
@Override
public boolean incrementToken() throws IOException {
if ( next() ) {
if (afterDiscard != null) {
int lg = afterDiscard.length();
afterDiscard.getChars(0, lg, termAtt.buffer(), 0);
termAtt.setLength(lg);
} else {
int lg = found.length();
found.get(termAtt.buffer(), 0, lg);
termAtt.setLength(lg);
}
return true;
} else {
return false;
}
}
private boolean next() {
if (currentWordIx >= words.length) {
return false;
}
if (!alwaysDiscard) {
// Last match was the discarded version,
// now do the non-discard version.
if (afterDiscard != null) {
afterDiscard = null;
return true;
}
}
afterDiscard = null;
if (charPos + 1 < currentWord.length && softMatcher.find(charPos+1)) {
charPos = softMatcher.end();
maybeDiscardHyphens();
return true;
} else {
return nextWord();
}
}
void maybeDiscardHyphens() {
found = CharBuffer.wrap(currentWord, charPos, currentWord.length - charPos);
Matcher discarding = discard.matcher(found);
if (discarding.find()) {
afterDiscard = discarding.replaceAll("");
}
}
private boolean nextWord() {
currentWordIx++;
if (currentWordIx >= words.length) {
return false;
}
currentWord = words[currentWordIx].toCharArray();
termAtt.resizeBuffer(currentWord.length);
charPos = 0;
softMatcher = subWordBoundary.matcher(words[currentWordIx]);
maybeDiscardHyphens();
return true;
}
}
static String getStringReaderContents(Reader reader) {
try {
reader.mark(Integer.MAX_VALUE);
int length = (int) reader.skip(Integer.MAX_VALUE);
reader.reset();
char fileContent[] = new char[length];
reader.read(fileContent);
reader.reset();
return new String(fileContent);
} catch (IOException e) {
throw new RuntimeException("Impossible",e);
}
}
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer source = new StandardTokenizer();
return new TokenStreamComponents(source){
private Reader reader;
@Override
protected void setReader(Reader reader) {
this.reader = reader;
super.setReader(reader);
}
@Override
public TokenStream getTokenStream() {
return new TermCompletionTokenStream(reader);
}
};
}
}