StopwordRefresh.java

/*
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bizosys.hsearch.inpipe.util;

import java.io.IOException;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.StopFilter;
import com.bizosys.oneline.ApplicationFault;
import com.bizosys.oneline.SystemFault;
import com.bizosys.oneline.services.batch.BatchTask;
import com.bizosys.oneline.util.StringUtils;

import com.bizosys.hsearch.common.IStorable;
import com.bizosys.hsearch.common.RecordScalar;
import com.bizosys.hsearch.common.Storable;
import com.bizosys.hsearch.hbase.HReader;
import com.bizosys.hsearch.hbase.HWriter;
import com.bizosys.hsearch.hbase.NV;
import com.bizosys.hsearch.inpipe.InpipeLog;
import com.bizosys.hsearch.schema.IOConstants;

/**
 * Scheduler stopword refresh task
 * @author karan
 *
 */
public class StopwordRefresh implements BatchTask {
	
	private static final char STOPWORD_SEPARATOR = '\t';
	private static byte[] STOP_WORD_LISTS_KEY = "STOP_WORDS".getBytes();

	public String getJobName() {
		return "StopWordRefresh";
	}

	public Object process() throws ApplicationFault, SystemFault {
		NV nv = new NV(IOConstants.NAME_VALUE_BYTES, IOConstants.NAME_VALUE_BYTES);
		RecordScalar scalar = new RecordScalar(STOP_WORD_LISTS_KEY,nv);
		HReader.getScalar(IOConstants.TABLE_CONFIG, scalar);
		if ( null != scalar.kv.data) {
			String words = new String(scalar.kv.data.toBytes());
			List<String> wordLst = StringUtils.fastSplit(words, STOPWORD_SEPARATOR);
			StopwordManager.getInstance().stopWords = buildStopwords(wordLst);
		}
		return null;
	}

	public void setJobName(String arg0) {
	}
	
	/**
	 * This refreshes the stopword list.
	 * @param allStopWords
	 * @return
	 * @throws ApplicationFault
	 */
	@SuppressWarnings("unchecked")
	private Set<String> buildStopwords(List<String> allStopWords) throws SystemFault{
		
		if ( null == allStopWords) {
			InpipeLog.l.warn(" FilterStopWords: No stop words." );
			return null;
		}
		
		try {
			Set wordSet = StopFilter.makeStopSet(allStopWords);
			if (InpipeLog.l.isInfoEnabled()) { 
				InpipeLog.l.info(" StopwordManager: stopWords.size - " + wordSet.size());
			}
			return (Set<String>) wordSet;

		} catch (Exception ex) {
			throw new SystemFault(ex);
		}
	}
	
	public static void add(List<String> lstWord) throws SystemFault{
		if ( null == lstWord) return;
		IStorable wordB = new Storable(
			StringUtils.listToString(lstWord, STOPWORD_SEPARATOR));
		NV nv = new NV(
			IOConstants.NAME_VALUE_BYTES, IOConstants.NAME_VALUE_BYTES,wordB);
		RecordScalar scalar = new RecordScalar(STOP_WORD_LISTS_KEY,nv);
		try {
			HWriter.insertScalar(IOConstants.TABLE_CONFIG, scalar);
		} catch (IOException ex) {
			InpipeLog.l.fatal("StopwordRefresh > ", ex);
			throw new SystemFault(ex);
		}
		
	}
}