/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sasi.analyzer;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.junit.Test;
import org.apache.cassandra.serializers.UTF8Serializer;
import static org.junit.Assert.assertEquals;
public class StandardAnalyzerTest
{
@Test
public void testTokenizationAscii() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/apache_license_header.txt");
StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder()
.maxTokenLength(5).build();
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(options);
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(is);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(67, tokens.size());
}
@Test
public void testTokenizationLoremIpsum() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/lorem_ipsum.txt");
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(is);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(62, tokens.size());
}
@Test
public void testTokenizationJaJp1() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/ja_jp_1.txt");
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
tokenizer.reset(is);
List<ByteBuffer> tokens = new ArrayList<>();
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(210, tokens.size());
}
@Test
public void testTokenizationJaJp2() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/ja_jp_2.txt");
StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder().stemTerms(true)
.ignoreStopTerms(true).alwaysLowerCaseTerms(true).build();
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(options);
tokenizer.reset(is);
List<ByteBuffer> tokens = new ArrayList<>();
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(57, tokens.size());
}
@Test
public void testTokenizationRuRu1() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/ru_ru_1.txt");
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(is);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(456, tokens.size());
}
@Test
public void testTokenizationZnTw1() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/zn_tw_1.txt");
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(is);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(963, tokens.size());
}
@Test
public void testTokenizationAdventuresOfHuckFinn() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/adventures_of_huckleberry_finn_mark_twain.txt");
StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder().stemTerms(true)
.ignoreStopTerms(true).useLocale(Locale.ENGLISH)
.alwaysLowerCaseTerms(true).build();
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(options);
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(is);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(37739, tokens.size());
}
@Test
public void testSkipStopWordBeforeStemmingFrench() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/french_skip_stop_words_before_stemming.txt");
StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder().stemTerms(true)
.ignoreStopTerms(true).useLocale(Locale.FRENCH)
.alwaysLowerCaseTerms(true).build();
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(options);
List<ByteBuffer> tokens = new ArrayList<>();
List<String> words = new ArrayList<>();
tokenizer.reset(is);
while (tokenizer.hasNext())
{
final ByteBuffer nextToken = tokenizer.next();
tokens.add(nextToken);
words.add(UTF8Serializer.instance.deserialize(nextToken.duplicate()));
}
assertEquals(4, tokens.size());
assertEquals("dans", words.get(0));
assertEquals("plui", words.get(1));
assertEquals("chanson", words.get(2));
assertEquals("connu", words.get(3));
}
@Test
public void tokenizeDomainNamesAndUrls() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/top_visited_domains.txt");
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
tokenizer.reset(is);
List<ByteBuffer> tokens = new ArrayList<>();
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(15, tokens.size());
}
@Test
public void testReuseAndResetTokenizerInstance() throws Exception
{
List<ByteBuffer> bbToTokenize = new ArrayList<>();
bbToTokenize.add(ByteBuffer.wrap("Nip it in the bud".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("I couldn’t care less".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("One and the same".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("The squeaky wheel gets the grease.".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("The pen is mightier than the sword.".getBytes()));
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(StandardTokenizerOptions.getDefaultOptions());
List<ByteBuffer> tokens = new ArrayList<>();
for (ByteBuffer bb : bbToTokenize)
{
tokenizer.reset(bb);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
}
assertEquals(10, tokens.size());
}
}