CSVAnalyzer.java

package org.apache.lucene.analysis.core;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.	See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.	You may obtain a copy of the License at
 *
 *		 http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.io.Reader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;

public final class CSVAnalyzer extends Analyzer {
	
	public CSVAnalyzer() {
	}
	
	@Override
	protected TokenStreamComponents createComponents(final String fieldName,
			final Reader reader) {
		return new TokenStreamComponents(new CSVTokenizer(reader));
	}
}

/*
 * 사용시 주의 : 컴마 뒤의 공백은 없어지나, 컴마 앞의 공백 즉, 단어 뒤에 공백이 있을시 사라지지 않음.		
 * */
final class CSVTokenizer extends Tokenizer {
	
	private static final char[] WHITESPACES = { ' ', '\t', '\n', '\r' };
	
	private char[] cbuf;
	private int currentPos;
	private int dataStarts;
	private int lastReaded;
	private int lastPos;
	private int baseOffset;
	private int nextOffset;
	
	private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
	private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);

	public CSVTokenizer(Reader in) {
		super(in);
		cbuf = new char[4096];
		baseOffset = 0;
		nextOffset = 0;
		lastPos = -1;
		dataStarts = -1;
	}

	@Override
	public void setReader(Reader input) throws IOException {
		super.setReader(input);
        lastReaded = 0;
		currentPos = 0;
		baseOffset = 0;
		nextOffset = 0;
		lastPos = -1;
		dataStarts = -1;
		this.clearAttributes();
	}
	
	private boolean isWhiteSpace(char c) {
		for (int inx = 0; inx < WHITESPACES.length; inx++) {
			if(WHITESPACES[inx] == c) {
				return true;
			}
		}
		return false;
	}
	
	@Override
	public boolean incrementToken() throws IOException {
	
		while(true) {
			logger.trace("current:{} / lastReaded:{} / dataStarts:{} / lastPos:{}", currentPos, lastReaded, dataStarts, lastPos);
			if(currentPos < lastReaded) {
				//read from buffer.
				if(dataStarts == -1) {
					if(lastPos == -1) {
						//ignore first blank.
						for (; currentPos < lastReaded;) {
							if (!isWhiteSpace(cbuf[currentPos])) {
								dataStarts = currentPos;
								lastPos = dataStarts;
								break;
							}
							currentPos++;
						}//for - removeWhitespace
					} else {
						dataStarts = 0;
					}
				}
					
				if(currentPos < lastReaded) {
					//comma detected. export char data
					if (cbuf[currentPos] == ',') {
						int length = 0;
						if (lastPos > dataStarts) {
							length = lastPos - dataStarts + 1;
							
							int startOffset = baseOffset + dataStarts;
							int endOffset = startOffset + length;
							
							termAttribute.copyBuffer(cbuf, dataStarts, length);
							offsetAttribute.setOffset(startOffset, endOffset);
							if(logger.isTraceEnabled()) {
								String term = new String(cbuf, dataStarts, length);
								logger.trace("output token:[{}] {}~{}", term, dataStarts, length);
							}
							lastPos = -1;
							dataStarts = -1;
							currentPos++;
							return true;
						} else {
							//init position
							dataStarts = -1;
							lastPos = -1;
						}
						
					} else {
						//for ignore whitespace at rear
						if (!isWhiteSpace(cbuf[currentPos])) {
							lastPos = currentPos;
						}
					}
					currentPos++;
				}
				//continue to first while
			} else {
				//if buffer not processed
				if(dataStarts != -1) {
					if(dataStarts < currentPos) {
						int length = currentPos - dataStarts;
						//shift data to head of array
						System.arraycopy(cbuf, dataStarts, cbuf, 0, length);
						dataStarts = 0;
						lastPos = length - (currentPos - lastPos);
						currentPos = length;
					}
				}
				
				if(currentPos == lastReaded) {
					currentPos = 0;
				}
				
				int rlen = 0;
				//read data from input
				rlen = input.read(cbuf, currentPos, cbuf.length - currentPos);
				
				if(logger.isTraceEnabled()) {
					logger.trace("read data..{} chars from {} / \"{}\"", rlen,
						currentPos, new String(cbuf, 0, currentPos + (rlen == -1 ? 0 : rlen)));
				}
				
				if(rlen == -1) {
					//no-more data
					break;
				}
				baseOffset = nextOffset;
				nextOffset += rlen;
				lastReaded = currentPos + rlen;
				continue;
			}
		}
		
		if(dataStarts != -1) {
			int length = lastPos - dataStarts + 1;
			int startOffset = baseOffset + dataStarts;
			int endOffset = startOffset + length;
			
			termAttribute.copyBuffer(cbuf, dataStarts, length);
			offsetAttribute.setOffset(startOffset, endOffset);
			if(logger.isTraceEnabled()) {
				String term = new String(cbuf, dataStarts, length);
				logger.trace("output token:[{}] {}~{}", term, dataStarts, length);
			}
			lastPos = -1;
			dataStarts = -1;
			lastReaded = -1;
			currentPos++;
			return true;
		}
		return false;
	}
}