/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.elasticsearch.indices.analysis; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.core.LetterTokenizer; import org.apache.lucene.analysis.core.LowerCaseTokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer; import org.apache.lucene.analysis.ngram.NGramTokenizer; import org.apache.lucene.analysis.path.PathHierarchyTokenizer; import org.apache.lucene.analysis.pattern.PatternTokenizer; import org.apache.lucene.analysis.standard.ClassicTokenizer; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.th.ThaiTokenizer; import org.elasticsearch.Version; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.analysis.MultiTermAwareComponent; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; import java.util.Locale; public enum PreBuiltTokenizers { STANDARD(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new StandardTokenizer(); } }, CLASSIC(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new ClassicTokenizer(); } }, UAX_URL_EMAIL(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new UAX29URLEmailTokenizer(); } }, PATH_HIERARCHY(CachingStrategy.ONE) { @Override protected Tokenizer create(Version version) { return new PathHierarchyTokenizer(); } }, KEYWORD(CachingStrategy.ONE) { @Override protected Tokenizer create(Version version) { return new KeywordTokenizer(); } }, LETTER(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new LetterTokenizer(); } }, LOWERCASE(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new LowerCaseTokenizer(); } @Override protected TokenFilterFactory getMultiTermComponent(Version version) { return PreBuiltTokenFilters.LOWERCASE.getTokenFilterFactory(version); } }, WHITESPACE(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new WhitespaceTokenizer(); } }, NGRAM(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new NGramTokenizer(); } }, EDGE_NGRAM(CachingStrategy.LUCENE) { @Override protected Tokenizer create(Version version) { return new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE); } }, PATTERN(CachingStrategy.ONE) { @Override protected Tokenizer create(Version version) { return new PatternTokenizer(Regex.compile("\\W+", null), -1); } }, THAI(CachingStrategy.ONE) { @Override protected Tokenizer create(Version version) { return new ThaiTokenizer(); } } ; protected abstract Tokenizer create(Version version); protected TokenFilterFactory getMultiTermComponent(Version version) { return null; } protected final PreBuiltCacheFactory.PreBuiltCache<TokenizerFactory> cache; PreBuiltTokenizers(CachingStrategy cachingStrategy) { cache = PreBuiltCacheFactory.getCache(cachingStrategy); } private interface MultiTermAwareTokenizerFactory extends TokenizerFactory, MultiTermAwareComponent {} public synchronized TokenizerFactory getTokenizerFactory(final Version version) { TokenizerFactory tokenizerFactory = cache.get(version); if (tokenizerFactory == null) { final String finalName = name().toLowerCase(Locale.ROOT); if (getMultiTermComponent(version) != null) { tokenizerFactory = new MultiTermAwareTokenizerFactory() { @Override public String name() { return finalName; } @Override public Tokenizer create() { return PreBuiltTokenizers.this.create(version); } @Override public Object getMultiTermComponent() { return PreBuiltTokenizers.this.getMultiTermComponent(version); } }; } else { tokenizerFactory = new TokenizerFactory() { @Override public String name() { return finalName; } @Override public Tokenizer create() { return PreBuiltTokenizers.this.create(version); } }; } cache.put(version, tokenizerFactory); } return tokenizerFactory; } /** * Get a pre built Tokenizer by its name or fallback to the default one * @param name Tokenizer name * @param defaultTokenizer default Tokenizer if name not found */ public static PreBuiltTokenizers getOrDefault(String name, PreBuiltTokenizers defaultTokenizer) { try { return valueOf(name.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { return defaultTokenizer; } } }