/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.cn.smart;
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;
/**
* Tokenizes input text into sentences.
* <p>
* The output tokens can then be broken into words with {@link WordTokenFilter}
* </p>
* <p><font color="#FF0000">
* WARNING: The status of the analyzers/smartcn <b>analysis.cn.smart</b> package is experimental.
* The APIs and file formats introduced here might change in the future and will not be
* supported anymore in such a case.</font>
* </p>
*/
public final class SentenceTokenizer extends Tokenizer {
/**
* End of sentence punctuation: 。,!?;,!?;
*/
private final static String PUNCTION = "。,!?;,!?;";
private final StringBuffer buffer = new StringBuffer();
private int tokenStart = 0, tokenEnd = 0;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private TypeAttribute typeAtt;
public SentenceTokenizer(Reader reader) {
super(reader);
init();
}
public SentenceTokenizer(AttributeSource source, Reader reader) {
super(source, reader);
init();
}
public SentenceTokenizer(AttributeFactory factory, Reader reader) {
super(factory, reader);
init();
}
private void init() {
termAtt = (TermAttribute) addAttribute(TermAttribute.class);
offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
}
public boolean incrementToken() throws IOException {
clearAttributes();
buffer.setLength(0);
int ci;
char ch, pch;
boolean atBegin = true;
tokenStart = tokenEnd;
ci = input.read();
ch = (char) ci;
while (true) {
if (ci == -1) {
break;
} else if (PUNCTION.indexOf(ch) != -1) {
// End of a sentence
buffer.append(ch);
tokenEnd++;
break;
} else if (atBegin && Utility.SPACES.indexOf(ch) != -1) {
tokenStart++;
tokenEnd++;
ci = input.read();
ch = (char) ci;
} else {
buffer.append(ch);
atBegin = false;
tokenEnd++;
pch = ch;
ci = input.read();
ch = (char) ci;
// Two spaces, such as CR, LF
if (Utility.SPACES.indexOf(ch) != -1
&& Utility.SPACES.indexOf(pch) != -1) {
// buffer.append(ch);
tokenEnd++;
break;
}
}
}
if (buffer.length() == 0)
return false;
else {
termAtt.setTermBuffer(buffer.toString());
offsetAtt.setOffset(correctOffset(tokenStart), correctOffset(tokenEnd));
typeAtt.setType("sentence");
return true;
}
}
public void reset() throws IOException {
super.reset();
tokenStart = tokenEnd = 0;
}
public void reset(Reader input) throws IOException {
super.reset(input);
reset();
}
public void end() throws IOException {
// set final offset
final int finalOffset = correctOffset(tokenEnd);
offsetAtt.setOffset(finalOffset, finalOffset);
}
}