// Copyright 2017 JanusGraph Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.janusgraph.diskstorage.solr;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import org.janusgraph.core.Cardinality;
import org.janusgraph.graphdb.internal.Order;
import org.janusgraph.core.JanusGraphElement;
import org.janusgraph.core.attribute.*;
import org.janusgraph.core.schema.Mapping;
import org.janusgraph.diskstorage.*;
import org.janusgraph.diskstorage.configuration.ConfigNamespace;
import org.janusgraph.diskstorage.configuration.ConfigOption;
import org.janusgraph.diskstorage.configuration.Configuration;
import org.janusgraph.diskstorage.indexing.*;
import org.janusgraph.diskstorage.solr.transform.GeoToWktConverter;
import org.janusgraph.diskstorage.util.DefaultTransaction;
import org.janusgraph.graphdb.configuration.PreInitializeConfigOptions;
import org.janusgraph.graphdb.database.serialize.AttributeUtil;
import org.janusgraph.graphdb.query.JanusGraphPredicate;
import org.janusgraph.graphdb.query.condition.*;
import org.janusgraph.graphdb.types.ParameterType;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.*;
import org.apache.solr.client.solrj.impl.*;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.*;
import java.util.AbstractMap.SimpleEntry;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.*;
/**
* @author Jared Holmberg (jholmberg@bericotechnoLogies.com), Pavel Yaskevich (pavel@thinkaurelius.com)
*/
@PreInitializeConfigOptions
public class SolrIndex implements IndexProvider {
private static final Logger logger = LoggerFactory.getLogger(SolrIndex.class);
private static final String DEFAULT_ID_FIELD = "id";
private enum Mode {
HTTP, CLOUD;
public static Mode parse(String mode) {
for (Mode m : Mode.values()) {
if (m.toString().equalsIgnoreCase(mode)) return m;
}
throw new IllegalArgumentException("Unrecognized mode: "+mode);
}
}
public static final ConfigNamespace SOLR_NS =
new ConfigNamespace(INDEX_NS, "solr", "Solr index configuration");
public static final ConfigOption<String> SOLR_MODE = new ConfigOption<String>(SOLR_NS,"mode",
"The operation mode for Solr which is either via HTTP (`http`) or using SolrCloud (`cloud`)",
ConfigOption.Type.GLOBAL_OFFLINE, "cloud");
public static final ConfigOption<Boolean> DYNAMIC_FIELDS = new ConfigOption<Boolean>(SOLR_NS,"dyn-fields",
"Whether to use dynamic fields (which appends the data type to the field name). If dynamic fields is disabled" +
"the user must map field names and define them explicitly in the schema.",
ConfigOption.Type.GLOBAL_OFFLINE, true);
public static final ConfigOption<String[]> KEY_FIELD_NAMES = new ConfigOption<String[]>(SOLR_NS,"key-field-names",
"Field name that uniquely identifies each document in Solr. Must be specified as a list of `collection=field`.",
ConfigOption.Type.GLOBAL, String[].class);
public static final ConfigOption<String> TTL_FIELD = new ConfigOption<String>(SOLR_NS,"ttl_field",
"Name of the TTL field for Solr collections.",
ConfigOption.Type.GLOBAL_OFFLINE, "ttl");
/** SolrCloud Configuration */
public static final ConfigOption<String> ZOOKEEPER_URL = new ConfigOption<String>(SOLR_NS,"zookeeper-url",
"URL of the Zookeeper instance coordinating the SolrCloud cluster",
ConfigOption.Type.MASKABLE, "localhost:2181");
public static final ConfigOption<Integer> NUM_SHARDS = new ConfigOption<Integer>(SOLR_NS,"num-shards",
"Number of shards for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
ConfigOption.Type.GLOBAL_OFFLINE, 1);
public static final ConfigOption<Integer> MAX_SHARDS_PER_NODE = new ConfigOption<Integer>(SOLR_NS,"max-shards-per-node",
"Maximum number of shards per node. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
ConfigOption.Type.GLOBAL_OFFLINE, 1);
public static final ConfigOption<Integer> REPLICATION_FACTOR = new ConfigOption<Integer>(SOLR_NS,"replication-factor",
"Replication factor for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
ConfigOption.Type.GLOBAL_OFFLINE, 1);
public static final ConfigOption<String> SOLR_DEFAULT_CONFIG = new ConfigOption<String>(SOLR_NS,"configset",
"If specified, the same solr configSet can be resued for each new Collection that is created in SolrCloud.",
ConfigOption.Type.MASKABLE, String.class);
/** HTTP Configuration */
public static final ConfigOption<String[]> HTTP_URLS = new ConfigOption<String[]>(SOLR_NS,"http-urls",
"List of URLs to use to connect to Solr Servers (LBHttpSolrClient is used), don't add core or collection name to the URL.",
ConfigOption.Type.MASKABLE, new String[] { "http://localhost:8983/solr" });
public static final ConfigOption<Integer> HTTP_CONNECTION_TIMEOUT = new ConfigOption<Integer>(SOLR_NS,"http-connection-timeout",
"Solr HTTP connection timeout.",
ConfigOption.Type.MASKABLE, 5000);
public static final ConfigOption<Boolean> HTTP_ALLOW_COMPRESSION = new ConfigOption<Boolean>(SOLR_NS,"http-compression",
"Enable/disable compression on the HTTP connections made to Solr.",
ConfigOption.Type.MASKABLE, false);
public static final ConfigOption<Integer> HTTP_MAX_CONNECTIONS_PER_HOST = new ConfigOption<Integer>(SOLR_NS,"http-max-per-host",
"Maximum number of HTTP connections per Solr host.",
ConfigOption.Type.MASKABLE, 20);
public static final ConfigOption<Integer> HTTP_GLOBAL_MAX_CONNECTIONS = new ConfigOption<Integer>(SOLR_NS,"http-max",
"Maximum number of HTTP connections in total to all Solr servers.",
ConfigOption.Type.MASKABLE, 100);
public static final ConfigOption<Boolean> WAIT_SEARCHER = new ConfigOption<Boolean>(SOLR_NS, "wait-searcher",
"When mutating - wait for the index to reflect new mutations before returning. This can have a negative impact on performance.",
ConfigOption.Type.LOCAL, false);
private static final IndexFeatures SOLR_FEATURES = new IndexFeatures.Builder().supportsDocumentTTL()
.setDefaultStringMapping(Mapping.TEXT).supportedStringMappings(Mapping.TEXT, Mapping.STRING).supportsCardinality(Cardinality.SINGLE).build();
private static Map<Geo, String> SPATIAL_PREDICATES = spatialPredicates();
private final SolrClient solrClient;
private final Configuration configuration;
private final Mode mode;
private final boolean dynFields;
private final Map<String, String> keyFieldIds;
private final String ttlField;
private final int maxResults;
private final boolean waitSearcher;
public SolrIndex(final Configuration config) throws BackendException {
Preconditions.checkArgument(config!=null);
configuration = config;
mode = Mode.parse(config.get(SOLR_MODE));
dynFields = config.get(DYNAMIC_FIELDS);
keyFieldIds = parseKeyFieldsForCollections(config);
maxResults = config.get(INDEX_MAX_RESULT_SET_SIZE);
ttlField = config.get(TTL_FIELD);
waitSearcher = config.get(WAIT_SEARCHER);
if (mode==Mode.CLOUD) {
String zookeeperUrl = config.get(SolrIndex.ZOOKEEPER_URL);
CloudSolrClient cloudServer = new CloudSolrClient(zookeeperUrl, true);
cloudServer.connect();
solrClient = cloudServer;
} else if (mode==Mode.HTTP) {
HttpClient clientParams = HttpClientUtil.createClient(new ModifiableSolrParams() {{
add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString());
add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, config.get(HTTP_CONNECTION_TIMEOUT).toString());
add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, config.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
add(HttpClientUtil.PROP_MAX_CONNECTIONS, config.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
}});
solrClient = new LBHttpSolrClient(clientParams, config.get(HTTP_URLS));
} else {
throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
}
}
private Map<String, String> parseKeyFieldsForCollections(Configuration config) throws BackendException {
Map<String, String> keyFieldNames = new HashMap<String, String>();
String[] collectionFieldStatements = config.has(KEY_FIELD_NAMES)?config.get(KEY_FIELD_NAMES):new String[0];
for (String collectionFieldStatement : collectionFieldStatements) {
String[] parts = collectionFieldStatement.trim().split("=");
if (parts.length != 2) {
throw new PermanentBackendException("Unable to parse the collection name / key field name pair. It should be of the format collection=field");
}
String collectionName = parts[0];
String keyFieldName = parts[1];
keyFieldNames.put(collectionName, keyFieldName);
}
return keyFieldNames;
}
private String getKeyFieldId(String collection) {
String field = keyFieldIds.get(collection);
if (field==null) field = DEFAULT_ID_FIELD;
return field;
}
/**
* Unlike the ElasticSearch Index, which is schema free, Solr requires a schema to
* support searching. This means that you will need to modify the solr schema with the
* appropriate field definitions in order to work properly. If you have a running instance
* of Solr and you modify its schema with new fields, don't forget to re-index!
* @param store Index store
* @param key New key to register
* @param information Datatype to register for the key
* @param tx enclosing transaction
* @throws org.janusgraph.diskstorage.BackendException
*/
@Override
public void register(String store, String key, KeyInformation information, BaseTransaction tx) throws BackendException {
if (mode==Mode.CLOUD) {
CloudSolrClient client = (CloudSolrClient) solrClient;
try {
createCollectionIfNotExists(client, configuration, store);
} catch (IOException e) {
throw new PermanentBackendException(e);
} catch (SolrServerException e) {
throw new PermanentBackendException(e);
} catch (InterruptedException e) {
throw new PermanentBackendException(e);
} catch (KeeperException e) {
throw new PermanentBackendException(e);
}
}
//Since all data types must be defined in the schema.xml, pre-registering a type does not work
}
@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
logger.debug("Mutating SOLR");
try {
for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
String collectionName = stores.getKey();
String keyIdField = getKeyFieldId(collectionName);
List<String> deleteIds = new ArrayList<String>();
Collection<SolrInputDocument> changes = new ArrayList<SolrInputDocument>();
for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
String docId = entry.getKey();
IndexMutation mutation = entry.getValue();
Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
//Handle any deletions
if (mutation.hasDeletions()) {
if (mutation.isDeleted()) {
logger.trace("Deleting entire document {}", docId);
deleteIds.add(docId);
} else {
HashSet<IndexEntry> fieldDeletions = Sets.newHashSet(mutation.getDeletions());
if (mutation.hasAdditions()) {
for (IndexEntry indexEntry : mutation.getAdditions()) {
fieldDeletions.remove(indexEntry);
}
}
deleteIndividualFieldsFromIndex(collectionName, keyIdField, docId, fieldDeletions);
}
}
if (mutation.hasAdditions()) {
int ttl = mutation.determineTTL();
SolrInputDocument doc = new SolrInputDocument();
doc.setField(keyIdField, docId);
boolean isNewDoc = mutation.isNew();
if (isNewDoc)
logger.trace("Adding new document {}", docId);
for (IndexEntry e : mutation.getAdditions()) {
final Object fieldValue = convertValue(e.value);
doc.setField(e.field, isNewDoc
? fieldValue : new HashMap<String, Object>(1) {{
put("set", fieldValue);
}});
}
if (ttl>0) {
Preconditions.checkArgument(isNewDoc,"Solr only supports TTL on new documents [%s]",docId);
doc.setField(ttlField, String.format("+%dSECONDS", ttl));
}
changes.add(doc);
}
}
commitDeletes(collectionName, deleteIds);
commitDocumentChanges(collectionName, changes);
}
} catch (IllegalArgumentException e) {
throw new PermanentBackendException("Unable to complete query on Solr.", e);
} catch (Exception e) {
throw storageException(e);
}
}
private Object convertValue(Object value) throws BackendException {
if (value instanceof Geoshape) {
return GeoToWktConverter.convertToWktString((Geoshape) value);
}
if (value instanceof UUID) {
return value.toString();
}
if(value instanceof Instant) {
if(Math.floorMod(((Instant) value).getNano(), 1000000) != 0) {
throw new IllegalArgumentException("Solr indexes do not support nanoseconds");
}
return new Date(((Instant) value).toEpochMilli());
}
return value;
}
@Override
public void restore(Map<String, Map<String, List<IndexEntry>>> documents, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
try {
for (Map.Entry<String, Map<String, List<IndexEntry>>> stores : documents.entrySet()) {
final String collectionName = stores.getKey();
List<String> deleteIds = new ArrayList<String>();
List<SolrInputDocument> newDocuments = new ArrayList<SolrInputDocument>();
for (Map.Entry<String, List<IndexEntry>> entry : stores.getValue().entrySet()) {
final String docID = entry.getKey();
final List<IndexEntry> content = entry.getValue();
if (content == null || content.isEmpty()) {
if (logger.isTraceEnabled())
logger.trace("Deleting document [{}]", docID);
deleteIds.add(docID);
continue;
}
newDocuments.add(new SolrInputDocument() {{
setField(getKeyFieldId(collectionName), docID);
for (IndexEntry addition : content) {
Object fieldValue = addition.value;
setField(addition.field, convertValue(fieldValue));
}
}});
}
commitDeletes(collectionName, deleteIds);
commitDocumentChanges(collectionName, newDocuments);
}
} catch (Exception e) {
throw new TemporaryBackendException("Could not restore Solr index", e);
}
}
private void deleteIndividualFieldsFromIndex(String collectionName, String keyIdField, String docId, HashSet<IndexEntry> fieldDeletions) throws SolrServerException, IOException {
if (fieldDeletions.isEmpty()) return;
Map<String, String> fieldDeletes = new HashMap<String, String>(1) {{ put("set", null); }};
SolrInputDocument doc = new SolrInputDocument();
doc.addField(keyIdField, docId);
StringBuilder sb = new StringBuilder();
for (IndexEntry fieldToDelete : fieldDeletions) {
doc.addField(fieldToDelete.field, fieldDeletes);
sb.append(fieldToDelete).append(",");
}
if (logger.isTraceEnabled())
logger.trace("Deleting individual fields [{}] for document {}", sb.toString(), docId);
UpdateRequest singleDocument = newUpdateRequest();
singleDocument.add(doc);
solrClient.request(singleDocument, collectionName);
}
private void commitDocumentChanges(String collectionName, Collection<SolrInputDocument> documents) throws SolrServerException, IOException {
if (documents.size() == 0) return;
try {
solrClient.request(newUpdateRequest().add(documents), collectionName);
} catch (HttpSolrClient.RemoteSolrException rse) {
logger.error("Unable to save documents to Solr as one of the shape objects stored were not compatible with Solr.", rse);
logger.error("Details in failed document batch: ");
for (SolrInputDocument d : documents) {
Collection<String> fieldNames = d.getFieldNames();
for (String name : fieldNames) {
logger.error(name + ":" + d.getFieldValue(name).toString());
}
}
throw rse;
}
}
private void commitDeletes(String collectionName, List<String> deleteIds) throws SolrServerException, IOException {
if (deleteIds.size() == 0) return;
solrClient.request(newUpdateRequest().deleteById(deleteIds), collectionName);
}
@Override
public List<String> query(IndexQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
List<String> result;
String collection = query.getStore();
String keyIdField = getKeyFieldId(collection);
SolrQuery solrQuery = new SolrQuery("*:*");
String queryFilter = buildQueryFilter(query.getCondition(), informations.get(collection));
solrQuery.addFilterQuery(queryFilter);
if (!query.getOrder().isEmpty()) {
List<IndexQuery.OrderEntry> orders = query.getOrder();
for (IndexQuery.OrderEntry order1 : orders) {
String item = order1.getKey();
SolrQuery.ORDER order = order1.getOrder() == Order.ASC ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
solrQuery.addSort(new SolrQuery.SortClause(item, order));
}
}
solrQuery.setStart(0);
if (query.hasLimit()) {
solrQuery.setRows(query.getLimit());
} else {
solrQuery.setRows(maxResults);
}
try {
QueryResponse response = solrClient.query(collection, solrQuery);
if (logger.isDebugEnabled())
logger.debug("Executed query [{}] in {} ms", query.getCondition(), response.getElapsedTime());
int totalHits = response.getResults().size();
if (!query.hasLimit() && totalHits >= maxResults)
logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
result = new ArrayList<String>(totalHits);
for (SolrDocument hit : response.getResults()) {
result.add(hit.getFieldValue(keyIdField).toString());
}
} catch (IOException e) {
logger.error("Query did not complete : ", e);
throw new PermanentBackendException(e);
} catch (SolrServerException e) {
logger.error("Unable to query Solr index.", e);
throw new PermanentBackendException(e);
}
return result;
}
@Override
public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
List<RawQuery.Result<String>> result;
String collection = query.getStore();
String keyIdField = getKeyFieldId(collection);
SolrQuery solrQuery = new SolrQuery(query.getQuery())
.addField(keyIdField)
.setIncludeScore(true)
.setStart(query.getOffset())
.setRows(query.hasLimit() ? query.getLimit() : maxResults);
try {
QueryResponse response = solrClient.query(collection, solrQuery);
if (logger.isDebugEnabled())
logger.debug("Executed query [{}] in {} ms", query.getQuery(), response.getElapsedTime());
int totalHits = response.getResults().size();
if (!query.hasLimit() && totalHits >= maxResults) {
logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
}
result = new ArrayList<RawQuery.Result<String>>(totalHits);
for (SolrDocument hit : response.getResults()) {
double score = Double.parseDouble(hit.getFieldValue("score").toString());
result.add(new RawQuery.Result<String>(hit.getFieldValue(keyIdField).toString(), score));
}
} catch (IOException e) {
logger.error("Query did not complete : ", e);
throw new PermanentBackendException(e);
} catch (SolrServerException e) {
logger.error("Unable to query Solr index.", e);
throw new PermanentBackendException(e);
}
return result;
}
private static String escapeValue(Object value) {
return ClientUtils.escapeQueryChars(value.toString());
}
public String buildQueryFilter(Condition<JanusGraphElement> condition, KeyInformation.StoreRetriever informations) {
if (condition instanceof PredicateCondition) {
PredicateCondition<String, JanusGraphElement> atom = (PredicateCondition<String, JanusGraphElement>) condition;
Object value = atom.getValue();
String key = atom.getKey();
JanusGraphPredicate janusgraphPredicate = atom.getPredicate();
if (value instanceof Number) {
String queryValue = escapeValue(value);
Preconditions.checkArgument(janusgraphPredicate instanceof Cmp, "Relation not supported on numeric types: " + janusgraphPredicate);
Cmp numRel = (Cmp) janusgraphPredicate;
switch (numRel) {
case EQUAL:
return (key + ":" + queryValue);
case NOT_EQUAL:
return ("-" + key + ":" + queryValue);
case LESS_THAN:
//use right curly to mean up to but not including value
return (key + ":[* TO " + queryValue + "}");
case LESS_THAN_EQUAL:
return (key + ":[* TO " + queryValue + "]");
case GREATER_THAN:
//use left curly to mean greater than but not including value
return (key + ":{" + queryValue + " TO *]");
case GREATER_THAN_EQUAL:
return (key + ":[" + queryValue + " TO *]");
default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
}
} else if (value instanceof String) {
Mapping map = getStringMapping(informations.get(key));
assert map==Mapping.TEXT || map==Mapping.STRING;
if (map==Mapping.TEXT && !janusgraphPredicate.toString().startsWith("CONTAINS"))
throw new IllegalArgumentException("Text mapped string values only support CONTAINS queries and not: " + janusgraphPredicate);
if (map==Mapping.STRING && janusgraphPredicate.toString().startsWith("CONTAINS"))
throw new IllegalArgumentException("String mapped string values do not support CONTAINS queries: " + janusgraphPredicate);
//Special case
if (janusgraphPredicate == Text.CONTAINS) {
//e.g. - if terms tomorrow and world were supplied, and fq=text:(tomorrow world)
//sample data set would return 2 documents: one where text = Tomorrow is the World,
//and the second where text = Hello World. Hence, we are decomposing the query string
//and building an AND query explicitly because we need AND semantics
value = ((String) value).toLowerCase();
List<String> terms = Text.tokenize((String) value);
if (terms.isEmpty()) {
return "";
} else if (terms.size() == 1) {
return (key + ":(" + escapeValue(terms.get(0)) + ")");
} else {
And<JanusGraphElement> andTerms = new And<JanusGraphElement>();
for (String term : terms) {
andTerms.add(new PredicateCondition<String, JanusGraphElement>(key, janusgraphPredicate, term));
}
return buildQueryFilter(andTerms, informations);
}
}
if (janusgraphPredicate == Text.PREFIX || janusgraphPredicate == Text.CONTAINS_PREFIX) {
return (key + ":" + escapeValue(value) + "*");
} else if (janusgraphPredicate == Text.REGEX || janusgraphPredicate == Text.CONTAINS_REGEX) {
return (key + ":/" + value + "/");
} else if (janusgraphPredicate == Cmp.EQUAL) {
return (key + ":\"" + escapeValue(value) + "\"");
} else if (janusgraphPredicate == Cmp.NOT_EQUAL) {
return ("-" + key + ":\"" + escapeValue(value) + "\"");
} else if (janusgraphPredicate == Text.FUZZY || janusgraphPredicate == Text.CONTAINS_FUZZY) {
return (key + ":"+escapeValue(value)+"~");
} else {
throw new IllegalArgumentException("Relation is not supported for string value: " + janusgraphPredicate);
}
} else if (value instanceof Geoshape) {
Mapping map = Mapping.getMapping(informations.get(key));
Preconditions.checkArgument(janusgraphPredicate instanceof Geo && janusgraphPredicate != Geo.DISJOINT, "Relation not supported on geo types: " + janusgraphPredicate);
Preconditions.checkArgument(map == Mapping.PREFIX_TREE || janusgraphPredicate == Geo.WITHIN || janusgraphPredicate == Geo.INTERSECT, "Relation not supported on geopoint types: " + janusgraphPredicate);
Geoshape geo = (Geoshape)value;
if (geo.getType() == Geoshape.Type.CIRCLE && (janusgraphPredicate == Geo.INTERSECT || map == Mapping.DEFAULT)) {
Geoshape.Point center = geo.getPoint();
return ("{!geofilt sfield=" + key +
" pt=" + center.getLatitude() + "," + center.getLongitude() +
" d=" + geo.getRadius() + "} distErrPct=0"); //distance in kilometers
} else if (geo.getType() == Geoshape.Type.BOX && (janusgraphPredicate == Geo.INTERSECT || map == Mapping.DEFAULT)) {
Geoshape.Point southwest = geo.getPoint(0);
Geoshape.Point northeast = geo.getPoint(1);
return (key + ":[" + southwest.getLatitude() + "," + southwest.getLongitude() +
" TO " + northeast.getLatitude() + "," + northeast.getLongitude() + "]");
} else if (map == Mapping.PREFIX_TREE) {
StringBuilder builder = new StringBuilder(key + ":\"");
builder.append(SPATIAL_PREDICATES.get((Geo) janusgraphPredicate) + "(");
builder.append(geo + ")\" distErrPct=0");
return builder.toString();
} else {
throw new IllegalArgumentException("Unsupported or invalid search shape type: " + geo.getType());
}
} else if (value instanceof Date || value instanceof Instant) {
String s = value.toString();
String queryValue = escapeValue(value instanceof Date ? toIsoDate((Date) value) : value.toString());
Preconditions.checkArgument(janusgraphPredicate instanceof Cmp, "Relation not supported on date types: " + janusgraphPredicate);
Cmp numRel = (Cmp) janusgraphPredicate;
switch (numRel) {
case EQUAL:
return (key + ":" + queryValue);
case NOT_EQUAL:
return ("-" + key + ":" + queryValue);
case LESS_THAN:
//use right curly to mean up to but not including value
return (key + ":[* TO " + queryValue + "}");
case LESS_THAN_EQUAL:
return (key + ":[* TO " + queryValue + "]");
case GREATER_THAN:
//use left curly to mean greater than but not including value
return (key + ":{" + queryValue + " TO *]");
case GREATER_THAN_EQUAL:
return (key + ":[" + queryValue + " TO *]");
default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
}
} else if (value instanceof Boolean) {
Cmp numRel = (Cmp) janusgraphPredicate;
String queryValue = escapeValue(value);
switch (numRel) {
case EQUAL:
return (key + ":" + queryValue);
case NOT_EQUAL:
return ("-" + key + ":" + queryValue);
default:
throw new IllegalArgumentException("Boolean types only support EQUAL or NOT_EQUAL");
}
} else if (value instanceof UUID) {
if (janusgraphPredicate == Cmp.EQUAL) {
return (key + ":\"" + escapeValue(value) + "\"");
} else if (janusgraphPredicate == Cmp.NOT_EQUAL) {
return ("-" + key + ":\"" + escapeValue(value) + "\"");
} else {
throw new IllegalArgumentException("Relation is not supported for uuid value: " + janusgraphPredicate);
}
} else throw new IllegalArgumentException("Unsupported type: " + value);
} else if (condition instanceof Not) {
String sub = buildQueryFilter(((Not)condition).getChild(),informations);
if (StringUtils.isNotBlank(sub)) return "-("+sub+")";
else return "";
} else if (condition instanceof And) {
int numChildren = ((And) condition).size();
StringBuilder sb = new StringBuilder();
for (Condition<JanusGraphElement> c : condition.getChildren()) {
String sub = buildQueryFilter(c, informations);
if (StringUtils.isBlank(sub))
continue;
// we don't have to add "+" which means AND iff
// a. it's a NOT query,
// b. expression is a single statement in the AND.
if (!sub.startsWith("-") && numChildren > 1)
sb.append("+");
sb.append(sub).append(" ");
}
return sb.toString();
} else if (condition instanceof Or) {
StringBuilder sb = new StringBuilder();
int element=0;
for (Condition<JanusGraphElement> c : condition.getChildren()) {
String sub = buildQueryFilter(c,informations);
if (StringUtils.isBlank(sub)) continue;
if (element==0) sb.append("(");
else sb.append(" OR ");
sb.append(sub);
element++;
}
if (element>0) sb.append(")");
return sb.toString();
} else {
throw new IllegalArgumentException("Invalid condition: " + condition);
}
}
private String toIsoDate(Date value) {
TimeZone tz = TimeZone.getTimeZone("UTC");
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
df.setTimeZone(tz);
return df.format(value);
}
/**
* Solr handles all transactions on the server-side. That means all
* commit, optimize, or rollback applies since the last commit/optimize/rollback.
* Solr documentation recommends best way to update Solr is in one process to avoid
* race conditions.
*
* @return New Transaction Handle
* @throws org.janusgraph.diskstorage.BackendException
*/
@Override
public BaseTransactionConfigurable beginTransaction(BaseTransactionConfig config) throws BackendException {
return new DefaultTransaction(config);
}
@Override
public void close() throws BackendException {
logger.trace("Shutting down connection to Solr", solrClient);
try {
solrClient.close();
} catch (IOException e) {
throw new TemporaryBackendException(e);
}
}
@Override
public void clearStorage() throws BackendException {
try {
if (mode!=Mode.CLOUD) throw new UnsupportedOperationException("Operation only supported for SolrCloud");
logger.debug("Clearing storage from Solr: {}", solrClient);
ZkStateReader zkStateReader = ((CloudSolrClient) solrClient).getZkStateReader();
zkStateReader.updateClusterState(true);
ClusterState clusterState = zkStateReader.getClusterState();
for (String collection : clusterState.getCollections()) {
logger.debug("Clearing collection [{}] in Solr",collection);
UpdateRequest deleteAll = newUpdateRequest();
deleteAll.deleteByQuery("*:*");
solrClient.request(deleteAll, collection);
}
} catch (SolrServerException e) {
logger.error("Unable to clear storage from index due to server error on Solr.", e);
throw new PermanentBackendException(e);
} catch (IOException e) {
logger.error("Unable to clear storage from index due to low-level I/O error.", e);
throw new PermanentBackendException(e);
} catch (Exception e) {
logger.error("Unable to clear storage from index due to general error.", e);
throw new PermanentBackendException(e);
}
}
@Override
public boolean supports(KeyInformation information, JanusGraphPredicate janusgraphPredicate) {
Class<?> dataType = information.getDataType();
Mapping mapping = Mapping.getMapping(information);
if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType) &&
!(mapping==Mapping.PREFIX_TREE && AttributeUtil.isGeo(dataType))) return false;
if(information.getCardinality() != Cardinality.SINGLE) {
return false;
}
if (Number.class.isAssignableFrom(dataType)) {
return janusgraphPredicate instanceof Cmp;
} else if (dataType == Geoshape.class) {
switch(mapping) {
case DEFAULT:
return janusgraphPredicate == Geo.WITHIN || janusgraphPredicate == Geo.INTERSECT;
case PREFIX_TREE:
return janusgraphPredicate == Geo.INTERSECT || janusgraphPredicate == Geo.WITHIN || janusgraphPredicate == Geo.CONTAINS;
}
} else if (AttributeUtil.isString(dataType)) {
switch(mapping) {
case DEFAULT:
case TEXT:
return janusgraphPredicate == Text.CONTAINS || janusgraphPredicate == Text.CONTAINS_PREFIX || janusgraphPredicate == Text.CONTAINS_REGEX || janusgraphPredicate == Text.CONTAINS_FUZZY;
case STRING:
return janusgraphPredicate == Cmp.EQUAL || janusgraphPredicate==Cmp.NOT_EQUAL || janusgraphPredicate==Text.REGEX || janusgraphPredicate==Text.PREFIX || janusgraphPredicate == Text.FUZZY;
// case TEXTSTRING:
// return (janusgraphPredicate instanceof Text) || janusgraphPredicate == Cmp.EQUAL || janusgraphPredicate==Cmp.NOT_EQUAL;
}
} else if (dataType == Date.class || dataType == Instant.class) {
if (janusgraphPredicate instanceof Cmp) return true;
} else if (dataType == Boolean.class) {
return janusgraphPredicate == Cmp.EQUAL || janusgraphPredicate == Cmp.NOT_EQUAL;
} else if (dataType == UUID.class) {
return janusgraphPredicate == Cmp.EQUAL || janusgraphPredicate==Cmp.NOT_EQUAL;
}
return false;
}
@Override
public boolean supports(KeyInformation information) {
if(information.getCardinality() != Cardinality.SINGLE) {
return false;
}
Class<?> dataType = information.getDataType();
Mapping mapping = Mapping.getMapping(information);
if (Number.class.isAssignableFrom(dataType) || dataType == Date.class || dataType == Instant.class || dataType == Boolean.class || dataType == UUID.class) {
if (mapping==Mapping.DEFAULT) return true;
} else if (AttributeUtil.isString(dataType)) {
if (mapping==Mapping.DEFAULT || mapping==Mapping.TEXT || mapping==Mapping.STRING) return true;
} else if (AttributeUtil.isGeo(dataType)) {
if (mapping==Mapping.DEFAULT || mapping==Mapping.PREFIX_TREE) return true;
}
return false;
}
@Override
public String mapKey2Field(String key, KeyInformation keyInfo) {
Preconditions.checkArgument(!StringUtils.containsAny(key, new char[]{' '}),"Invalid key name provided: %s",key);
if (!dynFields) return key;
if (ParameterType.MAPPED_NAME.hasParameter(keyInfo.getParameters())) return key;
String postfix;
Class datatype = keyInfo.getDataType();
if (AttributeUtil.isString(datatype)) {
Mapping map = getStringMapping(keyInfo);
switch (map) {
case TEXT: postfix = "_t"; break;
case STRING: postfix = "_s"; break;
default: throw new IllegalArgumentException("Unsupported string mapping: " + map);
}
} else if (AttributeUtil.isWholeNumber(datatype)) {
if (datatype.equals(Long.class)) postfix = "_l";
else postfix = "_i";
} else if (AttributeUtil.isDecimal(datatype)) {
if (datatype.equals(Float.class)) postfix = "_f";
else postfix = "_d";
} else if (datatype.equals(Geoshape.class)) {
postfix = "_g";
} else if (datatype.equals(Date.class) || datatype.equals(Instant.class)) {
postfix = "_dt";
} else if (datatype.equals(Boolean.class)) {
postfix = "_b";
} else if (datatype.equals(UUID.class)) {
postfix = "_uuid";
} else throw new IllegalArgumentException("Unsupported data type ["+datatype+"] for field: " + key);
return key+postfix;
}
@Override
public IndexFeatures getFeatures() {
return SOLR_FEATURES;
}
/*
################# UTILITY METHODS #######################
*/
private static Mapping getStringMapping(KeyInformation information) {
assert AttributeUtil.isString(information.getDataType());
Mapping map = Mapping.getMapping(information);
if (map==Mapping.DEFAULT) map = Mapping.TEXT;
return map;
}
private static Map<Geo, String> spatialPredicates() {
return Collections.unmodifiableMap(Stream.of(
new SimpleEntry<>(Geo.WITHIN, "IsWithin"),
new SimpleEntry<>(Geo.CONTAINS, "Contains"),
new SimpleEntry<>(Geo.INTERSECT, "Intersects"),
new SimpleEntry<>(Geo.DISJOINT, "IsDisjointTo"))
.collect(Collectors.toMap((e) -> e.getKey(), (e) -> e.getValue())));
}
private UpdateRequest newUpdateRequest() {
UpdateRequest req = new UpdateRequest();
if(waitSearcher) {
req.setAction(UpdateRequest.ACTION.COMMIT, true, true);
}
return req;
}
private BackendException storageException(Exception solrException) {
return new TemporaryBackendException("Unable to complete query on Solr.", solrException);
}
private static void createCollectionIfNotExists(CloudSolrClient client, Configuration config, String collection)
throws IOException, SolrServerException, KeeperException, InterruptedException {
if (!checkIfCollectionExists(client, collection)) {
Integer numShards = config.get(NUM_SHARDS);
Integer maxShardsPerNode = config.get(MAX_SHARDS_PER_NODE);
Integer replicationFactor = config.get(REPLICATION_FACTOR);
// Ideally this property used so a new configset is not uploaded for every single
// index (collection) created in solr.
// if a generic configSet is not set, make the configset name the same as the collection.
// This was the default behavior before a default configSet could be specified
String genericConfigSet = config.has(SOLR_DEFAULT_CONFIG) ? config.get(SOLR_DEFAULT_CONFIG):collection;
CollectionAdminRequest.Create createRequest = new CollectionAdminRequest.Create();
createRequest.setConfigName(genericConfigSet);
createRequest.setCollectionName(collection);
createRequest.setNumShards(numShards);
createRequest.setMaxShardsPerNode(maxShardsPerNode);
createRequest.setReplicationFactor(replicationFactor);
CollectionAdminResponse createResponse = createRequest.process(client);
if (createResponse.isSuccess()) {
logger.trace("Collection {} successfully created.", collection);
} else {
throw new SolrServerException(Joiner.on("\n").join(createResponse.getErrorMessages()));
}
}
waitForRecoveriesToFinish(client, collection);
}
/**
* Checks if the collection has already been created in Solr.
*/
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
ZkStateReader zkStateReader = server.getZkStateReader();
zkStateReader.updateClusterState(true);
ClusterState clusterState = zkStateReader.getClusterState();
return clusterState.getCollectionOrNull(collection) != null;
}
/**
* Wait for all the collection shards to be ready.
*/
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
ZkStateReader zkStateReader = server.getZkStateReader();
try {
boolean cont = true;
while (cont) {
boolean sawLiveRecovering = false;
zkStateReader.updateClusterState(true);
ClusterState clusterState = zkStateReader.getClusterState();
Map<String, Slice> slices = clusterState.getSlicesMap(collection);
Preconditions.checkNotNull("Could not find collection:" + collection, slices);
// change paths for Replica.State per Solr refactoring
// remove SYNC state per: http://tinyurl.com/pag6rwt
for (Map.Entry<String, Slice> entry : slices.entrySet()) {
Map<String, Replica> shards = entry.getValue().getReplicasMap();
for (Map.Entry<String, Replica> shard : shards.entrySet()) {
final String state = shard.getValue().getStr(ZkStateReader.STATE_PROP).toUpperCase();
if ((Replica.State.RECOVERING.name().equals(state) || Replica.State.DOWN.name().equals(state))
&& clusterState.liveNodesContain(shard.getValue().getStr(
ZkStateReader.NODE_NAME_PROP))) {
sawLiveRecovering = true;
}
}
}
if (!sawLiveRecovering) {
cont = false;
} else {
Thread.sleep(1000);
}
}
} finally {
logger.info("Exiting solr wait");
}
}
}