package org.apache.blur.server;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static org.apache.blur.utils.BlurConstants.BLUR_FIELDTYPE;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLURINDEX_CLASS;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_SIMILARITY;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_READ_INTERCEPTOR;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS;
import static org.apache.blur.utils.BlurConstants.SUPER;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.apache.blur.BlurConfiguration;
import org.apache.blur.analysis.FieldManager;
import org.apache.blur.analysis.FieldTypeDefinition;
import org.apache.blur.analysis.HdfsFieldManager;
import org.apache.blur.analysis.NoStopWordStandardAnalyzer;
import org.apache.blur.analysis.ThriftFieldManager;
import org.apache.blur.log.Log;
import org.apache.blur.log.LogFactory;
import org.apache.blur.lucene.search.FairSimilarity;
import org.apache.blur.lucene.security.index.AccessControlFactory;
import org.apache.blur.lucene.security.index.FilterAccessControlFactory;
import org.apache.blur.manager.ReadInterceptor;
import org.apache.blur.manager.writer.BlurIndex;
import org.apache.blur.manager.writer.BlurIndexConfig;
import org.apache.blur.manager.writer.BlurIndexSimpleWriter;
import org.apache.blur.thrift.generated.Blur.Iface;
import org.apache.blur.thrift.generated.ScoreType;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.blur.utils.BlurConstants;
import org.apache.blur.utils.BlurUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.similarities.Similarity;
//import org.apache.blur.manager.writer.BlurNRTIndex;
public class TableContext implements Cloneable {
private static final Log LOG = LogFactory.getLog(TableContext.class);
private static final String TYPES = "types";
private static ConcurrentHashMap<String, TableContext> _cache = new ConcurrentHashMap<String, TableContext>();
private static Configuration _systemConfiguration;
private static BlurConfiguration _systemBlurConfiguration;
private static final ReadInterceptor DEFAULT_INTERCEPTOR = new ReadInterceptor(null) {
@Override
public Filter getFilter() {
return null;
}
};
private Path _tablePath;
private String _defaultFieldName;
private String _table;
private IndexDeletionPolicy _indexDeletionPolicy;
private Similarity _similarity;
private Configuration _configuration;
private TableDescriptor _descriptor;
private long _timeBetweenCommits;
private long _timeBetweenRefreshs;
private ScoreType _defaultScoreType;
private Term _defaultPrimeDocTerm;
private FieldManager _fieldManager;
private BlurConfiguration _blurConfiguration;
private ReadInterceptor _readInterceptor;
private AccessControlFactory _accessControlFactory;
private Set<String> _discoverableFields;
protected TableContext() {
}
public static void clear() {
_cache.clear();
}
public static void clear(String table) {
_cache.remove(table);
}
public static TableContext create(TableDescriptor tableDescriptor) {
return create(tableDescriptor, false, null);
}
public static TableContext create(TableDescriptor tableDescriptor, boolean remote, Iface client) {
if (tableDescriptor == null) {
throw new NullPointerException("TableDescriptor can not be null.");
}
String name = tableDescriptor.getName();
if (name == null) {
throw new NullPointerException("Table name in the TableDescriptor can not be null.");
}
String tableUri = tableDescriptor.getTableUri();
if (tableUri == null) {
throw new NullPointerException("Table uri in the TableDescriptor can not be null.");
}
TableContext tableContext = _cache.get(name);
if (tableContext != null) {
return clone(tableDescriptor, tableContext);
}
synchronized (_cache) {
tableContext = _cache.get(name);
if (tableContext != null) {
return clone(tableDescriptor, tableContext);
}
return createInternal(tableDescriptor, remote, client, name, tableUri);
}
}
private static TableContext clone(TableDescriptor tableDescriptor, TableContext tableContext) {
TableContext clone = tableContext.clone();
TableDescriptor newTd = new TableDescriptor(clone._descriptor);
clone._descriptor = newTd;
clone._descriptor.setEnabled(tableDescriptor.isEnabled());
return clone;
}
private static TableContext createInternal(TableDescriptor tableDescriptor, boolean remote, Iface client,
String name, String tableUri) {
TableContext tableContext;
LOG.info("Creating table context for table [{0}]", name);
Configuration configuration = getSystemConfiguration();
BlurConfiguration blurConfiguration = getSystemBlurConfiguration();
Map<String, String> tableProperties = tableDescriptor.getTableProperties();
if (tableProperties != null) {
for (Entry<String, String> prop : tableProperties.entrySet()) {
configuration.set(prop.getKey(), prop.getValue());
blurConfiguration.set(prop.getKey(), prop.getValue());
}
}
tableContext = new TableContext();
tableContext._configuration = configuration;
tableContext._blurConfiguration = blurConfiguration;
tableContext._tablePath = new Path(tableUri);
tableContext._defaultFieldName = SUPER;
tableContext._table = name;
tableContext._descriptor = tableDescriptor;
tableContext._timeBetweenCommits = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_COMMITS, 60000);
tableContext._timeBetweenRefreshs = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS, 5000);
tableContext._defaultPrimeDocTerm = new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE);
tableContext._defaultScoreType = ScoreType.SUPER;
// TODO make configurable
tableContext._discoverableFields = new HashSet<String>(Arrays.asList(BlurConstants.ROW_ID, BlurConstants.RECORD_ID,
BlurConstants.FAMILY));
// TODO make configurable
tableContext._accessControlFactory = new FilterAccessControlFactory();
boolean strict = tableDescriptor.isStrictTypes();
String defaultMissingFieldType = tableDescriptor.getDefaultMissingFieldType();
boolean defaultMissingFieldLessIndexing = tableDescriptor.isDefaultMissingFieldLessIndexing();
Map<String, String> defaultMissingFieldProps = emptyIfNull(tableDescriptor.getDefaultMissingFieldProps());
Path storagePath = new Path(tableContext._tablePath, TYPES);
try {
FieldManager fieldManager;
if (remote) {
fieldManager = new ThriftFieldManager(SUPER, new NoStopWordStandardAnalyzer(), strict, defaultMissingFieldType,
defaultMissingFieldLessIndexing, defaultMissingFieldProps, configuration, client, name);
} else {
fieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath, configuration,
strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, defaultMissingFieldProps);
}
loadCustomTypes(tableContext, blurConfiguration, fieldManager);
fieldManager.loadFromStorage();
tableContext._fieldManager = fieldManager;
} catch (IOException e) {
throw new RuntimeException(e);
}
Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
KeepOnlyLastCommitDeletionPolicy.class);
tableContext._indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1, configuration),
tableContext);
Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY, FairSimilarity.class);
tableContext._similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext);
String readInterceptorClass = blurConfiguration.get(BLUR_SHARD_READ_INTERCEPTOR);
if (readInterceptorClass == null || readInterceptorClass.trim().isEmpty()) {
tableContext._readInterceptor = DEFAULT_INTERCEPTOR;
} else {
try {
@SuppressWarnings("unchecked")
Class<? extends ReadInterceptor> clazz = (Class<? extends ReadInterceptor>) Class.forName(readInterceptorClass);
Constructor<? extends ReadInterceptor> constructor = clazz
.getConstructor(new Class[] { BlurConfiguration.class });
tableContext._readInterceptor = constructor.newInstance(blurConfiguration);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
tableContext._similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext);
// DEFAULT_INTERCEPTOR
_cache.put(name, tableContext);
return tableContext.clone();
}
@SuppressWarnings("unchecked")
private static void loadCustomTypes(TableContext tableContext, BlurConfiguration blurConfiguration,
FieldManager fieldManager) {
Set<Entry<String, String>> entrySet = blurConfiguration.getProperties().entrySet();
TableDescriptor descriptor = tableContext._descriptor;
for (Entry<String, String> entry : entrySet) {
String key = entry.getKey();
String value = entry.getValue();
if (value == null || value.isEmpty()) {
continue;
}
if (key.startsWith(BLUR_FIELDTYPE)) {
String className = entry.getValue();
descriptor.putToTableProperties(key, className);
LOG.info("Attempting to load new type [{0}]", className);
Class<? extends FieldTypeDefinition> clazz;
try {
clazz = (Class<? extends FieldTypeDefinition>) Class.forName(className);
FieldTypeDefinition fieldTypeDefinition = clazz.newInstance();
fieldManager.registerType(clazz);
LOG.info("Sucessfully loaded new type [{0}] with name [{1}]", className, fieldTypeDefinition.getName());
} catch (ClassNotFoundException e) {
LOG.error("The field type definition class [{0}] was not found. Check the classpath.", e, className);
} catch (InstantiationException e) {
LOG.error("Could not create the field type definition [{0}].", e, className);
} catch (IllegalAccessException e) {
LOG.error("Unknown exception while trying to load field type definition [{0}].", e, className);
}
}
}
}
private static Map<String, String> emptyIfNull(Map<String, String> defaultMissingFieldProps) {
if (defaultMissingFieldProps == null) {
return new HashMap<String, String>();
}
return defaultMissingFieldProps;
}
private static Object configure(Object o, TableContext tableContext) {
if (o instanceof Configurable) {
((Configurable) o).setTableContext(tableContext);
}
return o;
}
public IndexDeletionPolicy getIndexDeletionPolicy() {
return _indexDeletionPolicy;
}
public Similarity getSimilarity() {
return _similarity;
}
public long getTimeBetweenCommits() {
return _timeBetweenCommits;
}
public long getTimeBetweenRefreshs() {
return _timeBetweenRefreshs;
}
public FieldManager getFieldManager() {
return _fieldManager;
}
public String getTable() {
return _table;
}
public Configuration getConfiguration() {
return _configuration;
}
public TableDescriptor getDescriptor() {
return _descriptor;
}
public Path getTablePath() {
return _tablePath;
}
public String getDefaultFieldName() {
return _defaultFieldName;
}
public Term getDefaultPrimeDocTerm() {
return _defaultPrimeDocTerm;
}
public ScoreType getDefaultScoreType() {
return _defaultScoreType;
}
public long getTimeBetweenWALSyncsNanos() {
return TimeUnit.MILLISECONDS.toNanos(10);
}
public BlurConfiguration getBlurConfiguration() {
return _blurConfiguration;
}
public static synchronized Configuration getSystemConfiguration() {
if (_systemConfiguration == null) {
_systemConfiguration = BlurUtil.newHadoopConfiguration(null);
}
return new Configuration(_systemConfiguration);
}
public static void setSystemConfiguration(Configuration systemConfiguration) {
TableContext._systemConfiguration = systemConfiguration;
}
public static synchronized BlurConfiguration getSystemBlurConfiguration() {
if (_systemBlurConfiguration == null) {
try {
_systemBlurConfiguration = new BlurConfiguration();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return _systemBlurConfiguration.clone();
}
public static void setSystemBlurConfiguration(BlurConfiguration systemBlurConfiguration) {
TableContext._systemBlurConfiguration = systemBlurConfiguration;
}
@SuppressWarnings("unchecked")
public BlurIndex newInstanceBlurIndex(BlurIndexConfig blurIndexConf) throws IOException {
String className = _blurConfiguration.get(BLUR_SHARD_BLURINDEX_CLASS, BlurIndexSimpleWriter.class.getName());
Class<? extends BlurIndex> clazz;
try {
clazz = (Class<? extends BlurIndex>) Class.forName(className);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
Constructor<? extends BlurIndex> constructor = findConstructor(clazz);
try {
return constructor.newInstance(blurIndexConf);
} catch (InstantiationException e) {
throw new IOException(e);
} catch (IllegalAccessException e) {
throw new IOException(e);
} catch (IllegalArgumentException e) {
throw new IOException(e);
} catch (InvocationTargetException e) {
throw new IOException(e);
}
}
private Constructor<? extends BlurIndex> findConstructor(Class<? extends BlurIndex> clazz) throws IOException {
try {
return clazz.getConstructor(new Class[] { BlurIndexConfig.class });
} catch (NoSuchMethodException e) {
throw new IOException(e);
} catch (SecurityException e) {
throw new IOException(e);
}
}
public ReadInterceptor getReadInterceptor() {
return _readInterceptor;
}
@Override
public TableContext clone() {
try {
return (TableContext) super.clone();
} catch (CloneNotSupportedException e) {
throw new RuntimeException(e);
}
}
public Set<String> getDiscoverableFields() {
return _discoverableFields;
}
public AccessControlFactory getAccessControlFactory() {
return _accessControlFactory;
}
}