ITokenizer.java

package tathya.text.tokenizer;

import java.util.List;

/**
 * This class represents the base class for all
 * tokenizers
 * @author anand
 */
public interface ITokenizer {

	/**
	 * Tokenize the given input string
	 * @param text	the text to be tokenized
	 * @return the list of tokens generated
	 */
	public List<String> tokenize(String text);
	
}