package org.apache.lucene.analysis.standard;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.analysis.*;
import org.apache.lucene.util.Version;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.util.Set;
/**
* Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
* LowerCaseFilter} and {@link StopFilter}, using a list of
* English stop words.
*
* <a name="version"/>
* <p>You must specify the required {@link Version}
* compatibility when creating StandardAnalyzer:
* <ul>
* <li> As of 2.9, StopFilter preserves position
* increments
* <li> As of 2.4, Tokens incorrectly identified as acronyms
* are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1608</a>
* </ul>
*
* @version $Id: StandardAnalyzer.java 829134 2009-10-23 17:18:53Z mikemccand $
*/
public class StandardAnalyzer extends Analyzer {
private Set stopSet;
/**
* Specifies whether deprecated acronyms should be replaced with HOST type.
* This is false by default to support backward compatibility.
*
* @deprecated this should be removed in the next release (3.0).
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
*/
private boolean replaceInvalidAcronym = defaultReplaceInvalidAcronym;
private static boolean defaultReplaceInvalidAcronym;
private boolean enableStopPositionIncrements;
// @deprecated
private boolean useDefaultStopPositionIncrements;
// Default to true (fixed the bug), unless the system prop is set
static {
final String v = System.getProperty("org.apache.lucene.analysis.standard.StandardAnalyzer.replaceInvalidAcronym");
if (v == null || v.equals("true"))
defaultReplaceInvalidAcronym = true;
else
defaultReplaceInvalidAcronym = false;
}
/**
*
* @return true if new instances of StandardTokenizer will
* replace mischaracterized acronyms
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
* @deprecated This will be removed (hardwired to true) in 3.0
*/
public static boolean getDefaultReplaceInvalidAcronym() {
return defaultReplaceInvalidAcronym;
}
/**
*
* @param replaceInvalidAcronym Set to true to have new
* instances of StandardTokenizer replace mischaracterized
* acronyms by default. Set to false to preserve the
* previous (before 2.4) buggy behavior. Alternatively,
* set the system property
* org.apache.lucene.analysis.standard.StandardAnalyzer.replaceInvalidAcronym
* to false.
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
* @deprecated This will be removed (hardwired to true) in 3.0
*/
public static void setDefaultReplaceInvalidAcronym(boolean replaceInvalidAcronym) {
defaultReplaceInvalidAcronym = replaceInvalidAcronym;
}
/** An array containing some common English words that are usually not
useful for searching.
@deprecated Use {@link #STOP_WORDS_SET} instead */
public static final String[] STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
/** An unmodifiable set containing some common English words that are usually not
useful for searching. */
public static final Set/*<String>*/ STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
/** Builds an analyzer with the default stop words ({@link
* #STOP_WORDS_SET}).
* @deprecated Use {@link #StandardAnalyzer(Version)} instead. */
public StandardAnalyzer() {
this(Version.LUCENE_24, STOP_WORDS_SET);
}
/** Builds an analyzer with the default stop words ({@link
* #STOP_WORDS}).
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
*/
public StandardAnalyzer(Version matchVersion) {
this(matchVersion, STOP_WORDS_SET);
}
/** Builds an analyzer with the given stop words.
* @deprecated Use {@link #StandardAnalyzer(Version, Set)}
* instead */
public StandardAnalyzer(Set stopWords) {
this(Version.LUCENE_24, stopWords);
}
/** Builds an analyzer with the given stop words.
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopWords stop words */
public StandardAnalyzer(Version matchVersion, Set stopWords) {
stopSet = stopWords;
init(matchVersion);
}
/** Builds an analyzer with the given stop words.
* @deprecated Use {@link #StandardAnalyzer(Version, Set)} instead */
public StandardAnalyzer(String[] stopWords) {
this(Version.LUCENE_24, StopFilter.makeStopSet(stopWords));
}
/** Builds an analyzer with the stop words from the given file.
* @see WordlistLoader#getWordSet(File)
* @deprecated Use {@link #StandardAnalyzer(Version, File)}
* instead
*/
public StandardAnalyzer(File stopwords) throws IOException {
this(Version.LUCENE_24, stopwords);
}
/** Builds an analyzer with the stop words from the given file.
* @see WordlistLoader#getWordSet(File)
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopwords File to read stop words from */
public StandardAnalyzer(Version matchVersion, File stopwords) throws IOException {
stopSet = WordlistLoader.getWordSet(stopwords);
init(matchVersion);
}
/** Builds an analyzer with the stop words from the given reader.
* @see WordlistLoader#getWordSet(Reader)
* @deprecated Use {@link #StandardAnalyzer(Version, Reader)}
* instead
*/
public StandardAnalyzer(Reader stopwords) throws IOException {
this(Version.LUCENE_24, stopwords);
}
/** Builds an analyzer with the stop words from the given reader.
* @see WordlistLoader#getWordSet(Reader)
* @param matchVersion Lucene version to match See {@link
* <a href="#version">above</a>}
* @param stopwords Reader to read stop words from */
public StandardAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
stopSet = WordlistLoader.getWordSet(stopwords);
init(matchVersion);
}
/**
*
* @param replaceInvalidAcronym Set to true if this analyzer should replace mischaracterized acronyms in the StandardTokenizer
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
*
* @deprecated Remove in 3.X and make true the only valid value
*/
public StandardAnalyzer(boolean replaceInvalidAcronym) {
this(Version.LUCENE_24, STOP_WORDS_SET);
this.replaceInvalidAcronym = replaceInvalidAcronym;
useDefaultStopPositionIncrements = true;
}
/**
* @param stopwords The stopwords to use
* @param replaceInvalidAcronym Set to true if this analyzer should replace mischaracterized acronyms in the StandardTokenizer
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
*
* @deprecated Remove in 3.X and make true the only valid value
*/
public StandardAnalyzer(Reader stopwords, boolean replaceInvalidAcronym) throws IOException{
this(Version.LUCENE_24, stopwords);
this.replaceInvalidAcronym = replaceInvalidAcronym;
}
/**
* @param stopwords The stopwords to use
* @param replaceInvalidAcronym Set to true if this analyzer should replace mischaracterized acronyms in the StandardTokenizer
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
*
* @deprecated Remove in 3.X and make true the only valid value
*/
public StandardAnalyzer(File stopwords, boolean replaceInvalidAcronym) throws IOException{
this(Version.LUCENE_24, stopwords);
this.replaceInvalidAcronym = replaceInvalidAcronym;
}
/**
*
* @param stopwords The stopwords to use
* @param replaceInvalidAcronym Set to true if this analyzer should replace mischaracterized acronyms in the StandardTokenizer
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
*
* @deprecated Remove in 3.X and make true the only valid value
*/
public StandardAnalyzer(String [] stopwords, boolean replaceInvalidAcronym) throws IOException{
this(Version.LUCENE_24, StopFilter.makeStopSet(stopwords));
this.replaceInvalidAcronym = replaceInvalidAcronym;
}
/**
* @param stopwords The stopwords to use
* @param replaceInvalidAcronym Set to true if this analyzer should replace mischaracterized acronyms in the StandardTokenizer
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
*
* @deprecated Remove in 3.X and make true the only valid value
*/
public StandardAnalyzer(Set stopwords, boolean replaceInvalidAcronym) throws IOException{
this(Version.LUCENE_24, stopwords);
this.replaceInvalidAcronym = replaceInvalidAcronym;
}
private final void init(Version matchVersion) {
setOverridesTokenStreamMethod(StandardAnalyzer.class);
if (matchVersion.onOrAfter(Version.LUCENE_29)) {
enableStopPositionIncrements = true;
} else {
useDefaultStopPositionIncrements = true;
}
if (matchVersion.onOrAfter(Version.LUCENE_24)) {
replaceInvalidAcronym = defaultReplaceInvalidAcronym;
} else {
replaceInvalidAcronym = false;
}
}
/** Constructs a {@link StandardTokenizer} filtered by a {@link
StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
public TokenStream tokenStream(String fieldName, Reader reader) {
StandardTokenizer tokenStream = new StandardTokenizer(reader, replaceInvalidAcronym);
tokenStream.setMaxTokenLength(maxTokenLength);
TokenStream result = new StandardFilter(tokenStream);
result = new LowerCaseFilter(result);
if (useDefaultStopPositionIncrements) {
result = new StopFilter(result, stopSet);
} else {
result = new StopFilter(enableStopPositionIncrements, result, stopSet);
}
return result;
}
private static final class SavedStreams {
StandardTokenizer tokenStream;
TokenStream filteredTokenStream;
}
/** Default maximum allowed token length */
public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
/**
* Set maximum allowed token length. If a token is seen
* that exceeds this length then it is discarded. This
* setting only takes effect the next time tokenStream or
* reusableTokenStream is called.
*/
public void setMaxTokenLength(int length) {
maxTokenLength = length;
}
/**
* @see #setMaxTokenLength
*/
public int getMaxTokenLength() {
return maxTokenLength;
}
/** @deprecated Use {@link #tokenStream} instead */
public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
if (overridesTokenStreamMethod) {
// LUCENE-1678: force fallback to tokenStream() if we
// have been subclassed and that subclass overrides
// tokenStream but not reusableTokenStream
return tokenStream(fieldName, reader);
}
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
setPreviousTokenStream(streams);
streams.tokenStream = new StandardTokenizer(reader);
streams.filteredTokenStream = new StandardFilter(streams.tokenStream);
streams.filteredTokenStream = new LowerCaseFilter(streams.filteredTokenStream);
if (useDefaultStopPositionIncrements) {
streams.filteredTokenStream = new StopFilter(streams.filteredTokenStream, stopSet);
} else {
streams.filteredTokenStream = new StopFilter(enableStopPositionIncrements, streams.filteredTokenStream, stopSet);
}
} else {
streams.tokenStream.reset(reader);
}
streams.tokenStream.setMaxTokenLength(maxTokenLength);
streams.tokenStream.setReplaceInvalidAcronym(replaceInvalidAcronym);
return streams.filteredTokenStream;
}
/**
*
* @return true if this Analyzer is replacing mischaracterized acronyms in the StandardTokenizer
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
* @deprecated This will be removed (hardwired to true) in 3.0
*/
public boolean isReplaceInvalidAcronym() {
return replaceInvalidAcronym;
}
/**
*
* @param replaceInvalidAcronym Set to true if this Analyzer is replacing mischaracterized acronyms in the StandardTokenizer
*
* See https://issues.apache.org/jira/browse/LUCENE-1068
* @deprecated This will be removed (hardwired to true) in 3.0
*/
public void setReplaceInvalidAcronym(boolean replaceInvalidAcronym) {
this.replaceInvalidAcronym = replaceInvalidAcronym;
}
}