/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.cn.smart;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.util.IOUtils;
public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase {
public void testChineseStopWordsDefault() throws Exception {
Analyzer ca = new SmartChineseAnalyzer(); /* will load stopwords */
String sentence = "我购买了道具和服装。";
String result[] = { "我", "购买", "了", "道具", "和", "服装" };
assertAnalyzesTo(ca, sentence, result);
ca.close();
// set stop-words from the outer world - must yield same behavior
ca = new SmartChineseAnalyzer(SmartChineseAnalyzer.getDefaultStopSet());
assertAnalyzesTo(ca, sentence, result);
ca.close();
}
/*
* This test is the same as the above, except with two phrases.
* This tests to ensure the SentenceTokenizer->WordTokenFilter chain works correctly.
*/
public void testChineseStopWordsDefaultTwoPhrases() throws Exception {
Analyzer ca = new SmartChineseAnalyzer(); /* will load stopwords */
String sentence = "我购买了道具和服装。 我购买了道具和服装。";
String result[] = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" };
assertAnalyzesTo(ca, sentence, result);
ca.close();
}
/*
* This test is the same as the above, except using an ideographic space as a separator.
* This tests to ensure the stopwords are working correctly.
*/
public void testChineseStopWordsDefaultTwoPhrasesIdeoSpace() throws Exception {
Analyzer ca = new SmartChineseAnalyzer(); /* will load stopwords */
String sentence = "我购买了道具和服装 我购买了道具和服装。";
String result[] = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" };
assertAnalyzesTo(ca, sentence, result);
ca.close();
}
/*
* Punctuation is handled in a strange way if you disable stopwords
* In this example the IDEOGRAPHIC FULL STOP is converted into a comma.
* if you don't supply (true) to the constructor, or use a different stopwords list,
* then punctuation is indexed.
*/
public void testChineseStopWordsOff() throws Exception {
Analyzer[] analyzers = new Analyzer[] {
new SmartChineseAnalyzer(false),/* doesn't load stopwords */
new SmartChineseAnalyzer(null) /* sets stopwords to empty set */};
String sentence = "我购买了道具和服装。";
String result[] = { "我", "购买", "了", "道具", "和", "服装", "," };
for (Analyzer analyzer : analyzers) {
assertAnalyzesTo(analyzer, sentence, result);
assertAnalyzesTo(analyzer, sentence, result);
}
IOUtils.close(analyzers);
}
/*
* Check that position increments after stopwords are correct,
* when stopfilter is configured with enablePositionIncrements
*/
public void testChineseStopWords2() throws Exception {
Analyzer ca = new SmartChineseAnalyzer(); /* will load stopwords */
String sentence = "Title:San"; // : is a stopword
String result[] = { "titl", "san"};
int startOffsets[] = { 0, 6 };
int endOffsets[] = { 5, 9 };
int posIncr[] = { 1, 2 };
assertAnalyzesTo(ca, sentence, result, startOffsets, endOffsets, posIncr);
ca.close();
}
public void testChineseAnalyzer() throws Exception {
Analyzer ca = new SmartChineseAnalyzer(true);
String sentence = "我购买了道具和服装。";
String[] result = { "我", "购买", "了", "道具", "和", "服装" };
assertAnalyzesTo(ca, sentence, result);
ca.close();
}
/*
* English words are lowercased and porter-stemmed.
*/
public void testMixedLatinChinese() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer(true);
assertAnalyzesTo(analyzer, "我购买 Tests 了道具和服装",
new String[] { "我", "购买", "test", "了", "道具", "和", "服装"});
analyzer.close();
}
/*
* Numerics are parsed as their own tokens
*/
public void testNumerics() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer(true);
assertAnalyzesTo(analyzer, "我购买 Tests 了道具和服装1234",
new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234"});
analyzer.close();
}
/*
* Full width alphas and numerics are folded to half-width
*/
public void testFullWidth() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer(true);
assertAnalyzesTo(analyzer, "我购买 Ｔｅｓｔｓ 了道具和服装１２３４",
new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234"});
analyzer.close();
}
/*
* Presentation form delimiters are removed
*/
public void testDelimiters() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer(true);
assertAnalyzesTo(analyzer, "我购买︱ Tests 了道具和服装",
new String[] { "我", "购买", "test", "了", "道具", "和", "服装"});
analyzer.close();
}
/*
* Text from writing systems other than Chinese and Latin are parsed as individual characters.
* (regardless of Unicode category)
*/
public void testNonChinese() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer(true);
assertAnalyzesTo(analyzer, "我购买 روبرتTests 了道具和服装",
new String[] { "我", "购买", "ر", "و", "ب", "ر", "ت", "test", "了", "道具", "和", "服装"});
analyzer.close();
}
/*
* Test what the analyzer does with out-of-vocabulary words.
* In this case the name is Yousaf Raza Gillani.
* Currently it is being analyzed into single characters...
*/
public void testOOV() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer(true);
assertAnalyzesTo(analyzer, "优素福·拉扎·吉拉尼",
new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" });
assertAnalyzesTo(analyzer, "优素福拉扎吉拉尼",
new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" });
analyzer.close();
}
public void testOffsets() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer(true);
assertAnalyzesTo(analyzer, "我购买了道具和服装",
new String[] { "我", "购买", "了", "道具", "和", "服装" },
new int[] { 0, 1, 3, 4, 6, 7 },
new int[] { 1, 3, 4, 6, 7, 9 });
analyzer.close();
}
public void testReusableTokenStream() throws Exception {
Analyzer a = new SmartChineseAnalyzer();
assertAnalyzesTo(a, "我购买 Tests 了道具和服装",
new String[] { "我", "购买", "test", "了", "道具", "和", "服装"},
new int[] { 0, 1, 4, 10, 11, 13, 14 },
new int[] { 1, 3, 9, 11, 13, 14, 16 });
assertAnalyzesTo(a, "我购买了道具和服装。",
new String[] { "我", "购买", "了", "道具", "和", "服装" },
new int[] { 0, 1, 3, 4, 6, 7 },
new int[] { 1, 3, 4, 6, 7, 9 });
a.close();
}
// LUCENE-3026
public void testLargeDocument() throws Exception {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 5000; i++) {
sb.append("我购买了道具和服装。");
}
try (Analyzer analyzer = new SmartChineseAnalyzer();
TokenStream stream = analyzer.tokenStream("", sb.toString())) {
stream.reset();
while (stream.incrementToken()) {
}
stream.end();
}
}
// LUCENE-3026
public void testLargeSentence() throws Exception {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 5000; i++) {
sb.append("我购买了道具和服装");
}
try (Analyzer analyzer = new SmartChineseAnalyzer();
TokenStream stream = analyzer.tokenStream("", sb.toString())) {
stream.reset();
while (stream.incrementToken()) {
}
stream.end();
}
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer();
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
analyzer.close();
}
/** blast some random large strings through the analyzer */
public void testRandomHugeStrings() throws Exception {
Analyzer analyzer = new SmartChineseAnalyzer();
checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192);
analyzer.close();
}
}