/*
* Data Hub Service (DHuS) - For Space data distribution.
* Copyright (C) 2013,2014,2015,2016 GAEL Systems
*
* This file is part of DHuS software sources.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.gael.dhus.search;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SuggesterResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.springframework.beans.factory.annotation.Autowired;
import fr.gael.dhus.database.object.config.search.GeocoderConfiguration;
import fr.gael.dhus.search.geocoder.CachedGeocoder;
import fr.gael.dhus.search.geocoder.Geocoder;
import fr.gael.dhus.search.geocoder.impl.NominatimGeocoder;
import fr.gael.dhus.system.config.ConfigurationManager;
/**
* Low level Solr interface.
*/
public class SolrDao
{
/** Logger. */
private static final Logger LOGGER = LogManager.getLogger(SolrDao.class);
/**
* Inner connections timeouts to the private solr service.
*/
public static final int INNER_TIMEOUT =
Integer.getInteger("dhus.search.innerTimeout", 5000);
/** URL path to solr service. */
private static final String SOLR_SVC = "/solr/dhus";
/** SolrJ client. */
private final HttpSolrClient solrClient;
/** Default Geocoder. */
private final Geocoder geocoder;
/** Dependency injection. */
@Autowired
private ConfigurationManager configurationManager;
/**
* DO NOT CALL! use Spring instead.
* @param geocoder_conf geocoder's configuration object.
*/
public SolrDao(GeocoderConfiguration geocoder_conf)
{
geocoder = new CachedGeocoder(new NominatimGeocoder(geocoder_conf));
solrClient = new HttpSolrClient("");
solrClient.setConnectionTimeout(INNER_TIMEOUT);
solrClient.setSoTimeout(INNER_TIMEOUT);
}
/**
* Initialises this DAO when the Tomcat server is started.
* (because getUrl() calls getPort() which delegates to TomcatServer).
*/
public void initServerStarted()
{
String dhus_url = configurationManager.getServerConfiguration().getLocalUrl();
solrClient.setBaseURL(dhus_url + SOLR_SVC);
}
/**
* System search.
* @param query a complete and well configured query.
* @return Solr response to given query.
* @throws SolrServerException a solr error occured.
* @throws IOException in case of network error.
*/
public QueryResponse search(SolrQuery query) throws SolrServerException, IOException
{
return solrClient.query(query);
}
/**
* Retrives SolrDocuments through a paginated iterator.
* Should not be used to delete documents because of the built-in lazy pagination.
* @param query to perform.
* @return an iterator on SolrDocument.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public Iterator<SolrDocument> scroll(SolrQuery query) throws IOException, SolrServerException
{
return new IterableSearchResult(solrClient, query);
}
/**
* Indexes a new document.
* @param doc to index.
* @return solr response.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public UpdateResponse index(SolrInputDocument doc) throws IOException, SolrServerException
{
return solrClient.add(doc);
}
/**
* Performs a batch index of _many_ documents, uses the ConcurrentUpdateSolrClient.
* <p>If you want faster indexing, disable the autoCommit and autoSoftCommit functionalities,
* see {@link #disableAutoCommit()}.
* @param source of document to index.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public void batchIndex(Iterator<SolrInputDocument> source) throws SolrServerException, IOException
{
String dhus_url = configurationManager.getServerConfiguration().getUrl() + SOLR_SVC;
try (ConcurrentUpdateSolrClient client = new ConcurrentUpdateSolrClient(dhus_url, 1000, 10))
{
client.add(source);
client.commit(true, true);
}
}
/**
* Get one document by its unique Id.
* @param id unique identifier.
* @return a doc or null if does not exist.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public SolrDocument getById(Long id) throws IOException, SolrServerException
{
return solrClient.getById(String.valueOf(id));
}
/**
* Deletes a SolrDocument with the given id.
* @param id of the SolrDocument to remove.
* @return solr response.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public UpdateResponse remove(long id) throws IOException, SolrServerException
{
UpdateResponse res = solrClient.deleteById(String.valueOf(id));
solrClient.commit(false, true, true); // mandatory explicit soft-commit.
return res;
}
/**
* Deletes all document, commit and optimizes the index.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public void removeAll() throws IOException, SolrServerException
{
// FIXME is it faster/more efficient to create a new empty core?
long etimedelete = solrClient.deleteByQuery("*:*").getElapsedTime();
long etimecommit = solrClient.commit(true, true, true).getElapsedTime();
long etimeoptimi = solrClient.optimize().getElapsedTime();
LOGGER.debug(String.format("removeAll: delete: %dms commit: %dms optimize: %dms",
etimedelete, etimecommit, etimeoptimi));
}
/**
* Get suggestions from the suggester component.
* @param input analysed by the suggester component.
* @return the suggester component response.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public SuggesterResponse getSuggestions(String input) throws IOException, SolrServerException
{
SolrQuery query = new SolrQuery(input);
query.setRequestHandler("/suggest");
return search(query).getSuggesterResponse();
}
/**
* Optimize the index, merges every segment of the index into one monolithic file.
* Optimizing is very expensive, and if the index is constantly changing,
* the slight performance boost will not last long...
* The tradeoff is not often worth it for a non static index.
* <p>
* Blocking method, will block until optimization is complete. Solr won't respond to
* search queries until optimization is done.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public void optimize() throws IOException, SolrServerException
{
solrClient.optimize();
}
/**
* Set the given properties and values using the config API.
* @param props properties to set.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public void setProperties(Map<String, String> props) throws SolrServerException, IOException
{
// Solrj does not support the config API yet.
StringBuilder command = new StringBuilder("{\"set-property\": {");
for (Map.Entry<String, String> entry: props.entrySet())
{
command.append('"').append(entry.getKey()).append('"').append(':');
command.append(entry.getValue()).append(',');
}
command.setLength(command.length()-1); // remove last comma
command.append("}}");
GenericSolrRequest rq = new GenericSolrRequest(SolrRequest.METHOD.POST, "/config", null);
ContentStream content = new ContentStreamBase.StringStream(command.toString());
rq.setContentStreams(Collections.singleton(content));
rq.process(solrClient);
}
/**
* Unset the given properties that have been previously set with {@link #setProperties(Map)}.
* @param props properties to unset.
* @throws IOException network error.
* @throws SolrServerException solr error.
*/
public void unsetProperties(Set<String> props) throws SolrServerException, IOException
{
// Solrj does not support the config API yet.
StringBuilder command = new StringBuilder("{\"unset-property\": [");
for (String prop: props)
{
command.append('"').append(prop).append('"').append(',');
}
command.setLength(command.length()-1); // remove last comma
command.append("]}");
GenericSolrRequest rq = new GenericSolrRequest(SolrRequest.METHOD.POST, "/config", null);
ContentStream content = new ContentStreamBase.StringStream(command.toString());
rq.setContentStreams(Collections.singleton(content));
rq.process(solrClient);
}
/**
* Geocode query.
* @param query query.
* @return result.
*/
public String updateQuery(String query)
{
for (String[]strs: SolrQueryParser.parse(query))
{
String key = strs[SolrQueryParser.INDEX_FIELD];
String token = strs[SolrQueryParser.INDEX_VALUE];
// If key defined, replace it by its lower case version.
if (!"".equals(key))
{
query = query.replace(key, key.toLowerCase());
}
if (!(!"".equals(key) ||
token.startsWith("{") ||
token.startsWith("[") ||
token.startsWith("(") ||
token.contains("*") ||
token.contains("?") ||
token.contains("TO") ||
token.contains("OR") ||
token.contains("AND") ||
token.matches(".*\\d.*")))
{
try
{
// If suggester knows the token: it is probably not a
// place location.
if (!getSuggestions(token).getSuggestions().get("suggest").
isEmpty())
throw new Exception();
}
catch (Exception e)
{
return query;
}
String wtk_boundaries = geocoder.getBoundariesWKT(token);
if (wtk_boundaries != null)
{
String locate = "(" + token +" OR footprint:\"Intersects(" +
wtk_boundaries + ") distErrPct=0\")";
query = query.replace(token, locate).trim();
}
}
}
return query;
}
/**
* An iterable SolrResponse, with pagination.
*/
private static class IterableSearchResult implements Iterator<SolrDocument>
{
/** Logger. */
private static final Logger LOGGER = LogManager.getLogger(IterableSearchResult.class);
/** Default fetch size of 50 solr documents. */
private static final int FETCH_SIZE = 50;
/** Solr client. */
private final SolrClient client;
/** Solr query. */
private final SolrQuery query;
/** For iretation purposes: offset in the current response. */
private int offset = 0;
/** Current response being served by this class. */
private QueryResponse rsp;
/**
* Creates a new SearchResult.
* @param client Solr client instance.
* @param query to perform.
* @throws SolrServerException Solr client exception.
* @throws IOException network exception.
*/
public IterableSearchResult(SolrClient client, SolrQuery query)
throws SolrServerException, IOException
{
Objects.requireNonNull(client);
Objects.requireNonNull(query);
this.client = client;
this.query = query;
this.query.setRows(FETCH_SIZE);
rsp = client.query(this.query, SolrRequest.METHOD.POST);
}
/** Run when every document in this.response have been served by next(). */
private void getNextResponse() {
int start = (this.query.getStart() != null)? this.query.getStart(): 0;
this.query.setStart(start + offset);
try
{
rsp = client.query(this.query, SolrRequest.METHOD.POST);
offset = 0;
}
catch (SolrServerException | IOException ex)
{
LOGGER.warn("An exception occured, no more solr document to serve", ex);
}
}
@Override
public boolean hasNext()
{
if (offset >= this.rsp.getResults().size())
{
getNextResponse();
}
return offset < this.rsp.getResults().size();
}
@Override
public SolrDocument next()
{
if (offset >= this.rsp.getResults().size())
{
getNextResponse();
if (offset >= this.rsp.getResults().size())
{
throw new NoSuchElementException();
}
}
int index = offset;
offset += 1;
return this.rsp.getResults().get(index);
}
@Override
public void remove()
{
throw new UnsupportedOperationException("Not implemented.");
}
}
}