/**
* Licensed to The Apereo Foundation under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
*
*
* The Apereo Foundation licenses this file to you under the Educational
* Community License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of the License
* at:
*
* http://opensource.org/licenses/ecl2.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*
*/
package org.opencastproject.scheduler.impl.solr;
import static org.opencastproject.scheduler.impl.Util.getEventIdentifier;
import static org.opencastproject.util.data.Option.option;
import org.opencastproject.metadata.dublincore.DCMIPeriod;
import org.opencastproject.metadata.dublincore.DublinCore;
import org.opencastproject.metadata.dublincore.DublinCoreCatalog;
import org.opencastproject.metadata.dublincore.DublinCoreCatalogList;
import org.opencastproject.metadata.dublincore.DublinCoreCatalogService;
import org.opencastproject.metadata.dublincore.DublinCoreValue;
import org.opencastproject.metadata.dublincore.EncodingSchemeUtils;
import org.opencastproject.metadata.dublincore.Temporal;
import org.opencastproject.scheduler.api.SchedulerQuery;
import org.opencastproject.scheduler.impl.SchedulerServiceDatabaseException;
import org.opencastproject.scheduler.impl.SchedulerServiceIndex;
import org.opencastproject.solr.SolrServerFactory;
import org.opencastproject.util.NotFoundException;
import org.opencastproject.util.SolrUtils;
import org.opencastproject.util.data.Tuple;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.osgi.service.component.ComponentContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* Implements {@link SchedulerServiceIndex}.
*/
public class SchedulerServiceSolrIndex implements SchedulerServiceIndex {
/** The logger */
private static final Logger logger = LoggerFactory.getLogger(SchedulerServiceSolrIndex.class);
/** Configuration key for a remote solr server */
public static final String CONFIG_SOLR_URL = "org.opencastproject.scheduler.solr.url";
/** Configuration key for an embedded solr configuration and data directory */
public static final String CONFIG_SOLR_ROOT = "org.opencastproject.scheduler.solr.dir";
/** Delimeter used for concatenating multivalued fields for sorting fields in solr */
public static final String SOLR_MULTIVALUED_DELIMETER = "; ";
/** Connection to the solr server. Solr is used to search for events. The event data are stored as xml files. */
private SolrServer solrServer = null;
/** The root directory to use for solr config and data files */
protected String solrRoot = null;
/** The URL to connect to a remote solr server */
protected URL solrServerUrl = null;
/** Dublin core service */
protected DublinCoreCatalogService dcService;
/** Whether indexing is synchronous or asynchronous */
protected boolean synchronousIndexing;
/** Executor used for asynchronous indexing */
protected ExecutorService indexingExecutor;
/**
* No-argument constructor for OSGi declarative services.
*/
public SchedulerServiceSolrIndex() {
}
/**
* Constructs Solr index with specified storage directory.
*/
public SchedulerServiceSolrIndex(String storageDirectory) {
solrRoot = storageDirectory;
}
/**
* OSGi callback for setting Dublin core service.
*
* @param dcService
* {@link DublinCoreCatalogService} object
*/
public void setDublinCoreService(DublinCoreCatalogService dcService) {
this.dcService = dcService;
}
/**
* Callback from the OSGi environment on component registration. Retrieves location of the solr index.
*
* @param cc
* the component context
*/
public void activate(ComponentContext cc) {
if (cc == null) {
if (solrRoot == null)
throw new IllegalStateException("Storage dir must be set");
// default to synchronous indexing
synchronousIndexing = true;
} else {
String solrServerUrlConfig = StringUtils.trimToNull(cc.getBundleContext().getProperty(CONFIG_SOLR_URL));
if (solrServerUrlConfig != null) {
try {
solrServerUrl = new URL(solrServerUrlConfig);
} catch (MalformedURLException e) {
throw new IllegalStateException("Unable to connect to solr at " + solrServerUrlConfig, e);
}
} else {
solrRoot = SolrServerFactory.getEmbeddedDir(cc, CONFIG_SOLR_ROOT, "scheduler");
}
Object syncIndexingConfig = cc.getProperties().get("synchronousIndexing");
if ((syncIndexingConfig != null) && ((syncIndexingConfig instanceof Boolean))) {
synchronousIndexing = ((Boolean) syncIndexingConfig).booleanValue();
} else {
synchronousIndexing = true;
}
}
activate();
}
/**
* OSGi callback for deactivation.
*
* @param cc
* the component context
*/
public void deactivate(ComponentContext cc) {
deactivate();
}
@Override
public void activate() {
// Set up the solr server
if (solrServerUrl != null) {
solrServer = SolrServerFactory.newRemoteInstance(solrServerUrl);
} else {
try {
setupSolr(new File(solrRoot));
} catch (IOException e) {
throw new IllegalStateException("Unable to connect to solr at " + solrRoot, e);
} catch (SolrServerException e) {
throw new IllegalStateException("Unable to connect to solr at " + solrRoot, e);
}
}
// set up indexing
if (this.synchronousIndexing) {
logger.debug("Events will be added to the search index synchronously");
} else {
logger.debug("Events will be added to the search index asynchronously");
indexingExecutor = Executors.newSingleThreadExecutor();
}
}
/**
* Prepares the embedded solr environment.
*
* @param solrRoot
* the solr root directory
*/
public void setupSolr(File solrRoot) throws IOException, SolrServerException {
logger.debug("Setting up solr search index at {}", solrRoot);
File solrConfigDir = new File(solrRoot, "conf");
// Create the config directory
if (solrConfigDir.exists()) {
logger.debug("solr search index found at {}", solrConfigDir);
} else {
logger.debug("solr config directory doesn't exist. Creating {}", solrConfigDir);
FileUtils.forceMkdir(solrConfigDir);
}
// Make sure there is a configuration in place
copyClasspathResourceToFile("/solr/conf/protwords.txt", solrConfigDir);
copyClasspathResourceToFile("/solr/conf/schema.xml", solrConfigDir);
copyClasspathResourceToFile("/solr/conf/scripts.conf", solrConfigDir);
copyClasspathResourceToFile("/solr/conf/solrconfig.xml", solrConfigDir);
copyClasspathResourceToFile("/solr/conf/stopwords.txt", solrConfigDir);
copyClasspathResourceToFile("/solr/conf/synonyms.txt", solrConfigDir);
// Test for the existence of a data directory
File solrDataDir = new File(solrRoot, "data");
if (!solrDataDir.exists()) {
FileUtils.forceMkdir(solrDataDir);
}
// Test for the existence of the index. Note that an empty index directory will prevent solr from
// completing normal setup.
File solrIndexDir = new File(solrDataDir, "index");
if (solrIndexDir.isDirectory() && solrIndexDir.list().length == 0) {
FileUtils.deleteDirectory(solrIndexDir);
}
solrServer = SolrServerFactory.newEmbeddedInstance(solrRoot, solrDataDir);
}
@Override
public void deactivate() {
SolrServerFactory.shutdown(solrServer);
}
// TODO: generalize this method
private void copyClasspathResourceToFile(String classpath, File dir) {
InputStream in = null;
FileOutputStream fos = null;
try {
in = SchedulerServiceSolrIndex.class.getResourceAsStream(classpath);
File file = new File(dir, FilenameUtils.getName(classpath));
logger.debug("copying " + classpath + " to " + file);
fos = new FileOutputStream(file);
IOUtils.copy(in, fos);
} catch (IOException e) {
throw new RuntimeException("Error copying solr classpath resource to the filesystem", e);
} finally {
IOUtils.closeQuietly(in);
IOUtils.closeQuietly(fos);
}
}
@Override
public void index(DublinCoreCatalog dc) throws SchedulerServiceDatabaseException {
index(dc, null);
}
@Override
public void index(DublinCoreCatalog dc, Properties captureAgentProperties) throws SchedulerServiceDatabaseException {
// See if we are updating
SolrDocument retrievedDoc = retrieveDocumentById(getEventIdentifier(dc));
// Prepare the new document
final SolrInputDocument doc = createDocument(dc);
String serializedCAProperties = null;
boolean optOut = false;
boolean blacklisted = false;
if (retrievedDoc != null) {
serializedCAProperties = (String) retrievedDoc.getFirstValue(SolrFields.CA_PROPERTIES);
optOut = BooleanUtils.toBoolean((Boolean) retrievedDoc.getFirstValue(SolrFields.OPT_OUT));
blacklisted = BooleanUtils.toBoolean((Boolean) retrievedDoc.getFirstValue(SolrFields.BLACKLISTED));
}
// Use updated capture agent properties if available. Otherwise, read from the existing doc
if (captureAgentProperties != null) {
try {
serializedCAProperties = serializeProperties(captureAgentProperties);
} catch (IOException e) {
logger.error("Error serializing capture agent properties: {}", e.getMessage());
throw new SchedulerServiceDatabaseException(e);
}
}
// doc.setField(SolrFields.OPT_OUT, optOut);
// doc.setField(SolrFields.BLACKLISTED, blacklisted);
doc.setField(SolrFields.CA_PROPERTIES, serializedCAProperties);
doc.setField(SolrFields.LAST_MODIFIED, new Date());
if (synchronousIndexing) {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
throw new SchedulerServiceDatabaseException("Unable to index event", e);
}
} else {
indexingExecutor.submit(new Runnable() {
@Override
public void run() {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
logger.warn("Unable to index event {}: {}", doc.getFieldValue(SolrFields.ID_KEY), e.getMessage());
}
}
});
}
}
@Override
public void index(final long eventId, Properties captureAgentProperties) throws NotFoundException,
SchedulerServiceDatabaseException {
SolrDocument result = retrieveDocumentById(eventId);
if (result == null) {
logger.warn("No event exists with event ID {}", eventId);
throw new NotFoundException("Event with ID " + eventId + " does not exist.");
}
String serializedCA = null;
// Use updated capture agent properties if available. Otherwise, read from the existing doc
if (captureAgentProperties != null) {
try {
serializedCA = serializeProperties(captureAgentProperties);
} catch (IOException e) {
logger.error("Error serializing capture agent properties for event '{}': {}", eventId, e.getMessage());
throw new SchedulerServiceDatabaseException(e);
}
} else {
serializedCA = (String) result.getFirstValue(SolrFields.CA_PROPERTIES);
}
final SolrInputDocument doc = ClientUtils.toSolrInputDocument(result);
doc.setField(SolrFields.CA_PROPERTIES, serializedCA);
doc.setField(SolrFields.LAST_MODIFIED, new Date());
if (synchronousIndexing) {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
throw new SchedulerServiceDatabaseException("Unable to index event capture agent metadata", e);
}
} else {
indexingExecutor.submit(new Runnable() {
@Override
public void run() {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
logger.warn("Unable to update event {} capture agent metadata: {}", eventId, e.getMessage());
}
}
});
}
}
@Override
public void indexOptOut(final long eventId, boolean optOut) throws NotFoundException,
SchedulerServiceDatabaseException {
SolrDocument result = retrieveDocumentById(eventId);
if (result == null) {
logger.warn("No event exists with event ID {}", eventId);
throw new NotFoundException("Event with ID " + eventId + " does not exist.");
}
final SolrInputDocument doc = ClientUtils.toSolrInputDocument(result);
doc.setField(SolrFields.OPT_OUT, optOut);
doc.setField(SolrFields.LAST_MODIFIED, new Date());
if (synchronousIndexing) {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
throw new SchedulerServiceDatabaseException("Unable to index event opt out status", e);
}
} else {
indexingExecutor.submit(new Runnable() {
@Override
public void run() {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
logger.warn("Unable to update event {} opt out status: {}", eventId, ExceptionUtils.getStackTrace(e));
}
}
});
}
}
@Override
public void indexBlacklisted(final long eventId, boolean blacklisted) throws NotFoundException,
SchedulerServiceDatabaseException {
SolrDocument result = retrieveDocumentById(eventId);
if (result == null) {
logger.warn("No event exists with event ID {}", eventId);
throw new NotFoundException("Event with ID " + eventId + " does not exist.");
}
final SolrInputDocument doc = ClientUtils.toSolrInputDocument(result);
doc.setField(SolrFields.BLACKLISTED, blacklisted);
doc.setField(SolrFields.LAST_MODIFIED, new Date());
if (synchronousIndexing) {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
throw new SchedulerServiceDatabaseException("Unable to index event blacklist status", e);
}
} else {
indexingExecutor.submit(new Runnable() {
@Override
public void run() {
try {
synchronized (solrServer) {
solrServer.add(doc);
solrServer.commit();
}
} catch (Exception e) {
logger.warn("Unable to update event {} blacklist status: {}", eventId, ExceptionUtils.getStackTrace(e));
}
}
});
}
}
/**
* Creates solr document for inserting into solr index.
*
* @param dc
* {@link DublinCoreCatalog} to be stored in index
* @return {@link SolrInputDocument} created out of Dublin core
*/
protected SolrInputDocument createDocument(DublinCoreCatalog dc) {
final SolrInputDocument doc = new SolrInputDocument();
doc.setField(SolrFields.ID_KEY, dc.getFirst(DublinCore.PROPERTY_IDENTIFIER));
try {
doc.setField(SolrFields.XML_KEY, serializeDublinCore(dc));
} catch (IOException e1) {
throw new IllegalArgumentException(e1);
}
// single valued fields
if (dc.hasValue(DublinCore.PROPERTY_TITLE)) {
doc.setField(SolrFields.TITLE_KEY, dc.getFirst(DublinCore.PROPERTY_TITLE));
}
if (dc.hasValue(DublinCore.PROPERTY_CREATED)) {
Temporal temporal = EncodingSchemeUtils.decodeTemporal(dc.get(DublinCore.PROPERTY_CREATED).get(0));
temporal.fold(new Temporal.Match<Void>() {
@Override
public Void period(DCMIPeriod period) {
doc.setField(SolrFields.CREATED_KEY, period.getStart());
return null;
}
@Override
public Void instant(Date instant) {
doc.setField(SolrFields.CREATED_KEY, instant);
return null;
}
@Override
public Void duration(long duration) {
throw new IllegalArgumentException("Dublin core dc:created is neither a date nor a period");
}
});
}
if (dc.hasValue(DublinCore.PROPERTY_AVAILABLE)) {
Temporal temporal = EncodingSchemeUtils.decodeTemporal(dc.get(DublinCore.PROPERTY_AVAILABLE).get(0));
temporal.fold(new Temporal.Match<Void>() {
@Override
public Void period(DCMIPeriod period) {
if (period.hasStart()) {
doc.setField(SolrFields.AVAILABLE_FROM_KEY, period.getStart());
}
if (period.hasEnd()) {
doc.setField(SolrFields.AVAILABLE_TO_KEY, period.getEnd());
}
return null;
}
@Override
public Void instant(Date instant) {
doc.setField(SolrFields.AVAILABLE_FROM_KEY, instant);
return null;
}
@Override
public Void duration(long duration) {
throw new IllegalArgumentException("Dublin core field dc:available is neither a date nor a period");
}
});
}
if (dc.hasValue(DublinCore.PROPERTY_TEMPORAL)) {
Temporal temporal = EncodingSchemeUtils.decodeTemporal(dc.get(DublinCore.PROPERTY_TEMPORAL).get(0));
temporal.fold(new Temporal.Match<Void>() {
@Override
public Void period(DCMIPeriod period) {
if (period.hasStart()) {
doc.setField(SolrFields.STARTS_KEY, SolrUtils.serializeDate(period.getStart()));
}
if (period.hasEnd()) {
doc.setField(SolrFields.ENDS_KEY, SolrUtils.serializeDate(period.getEnd()));
}
return null;
}
@Override
public Void instant(Date instant) {
doc.setField(SolrFields.STARTS_KEY, instant);
return null;
}
@Override
public Void duration(long duration) {
throw new IllegalArgumentException("Dublin core field dc:temporal is neither a date nor a period");
}
});
}
// multivalued fields
addMultiValuedFieldToSolrDocument(doc, SolrFields.SUBJECT_KEY, dc.get(DublinCore.PROPERTY_SUBJECT));
addMultiValuedFieldToSolrDocument(doc, SolrFields.CREATOR_KEY, dc.get(DublinCore.PROPERTY_CREATOR));
addMultiValuedFieldToSolrDocument(doc, SolrFields.PUBLISHER_KEY, dc.get(DublinCore.PROPERTY_PUBLISHER));
addMultiValuedFieldToSolrDocument(doc, SolrFields.CONTRIBUTOR_KEY, dc.get(DublinCore.PROPERTY_CONTRIBUTOR));
addMultiValuedFieldToSolrDocument(doc, SolrFields.ABSTRACT_KEY, dc.get(DublinCore.PROPERTY_ABSTRACT));
addMultiValuedFieldToSolrDocument(doc, SolrFields.DESCRIPTION_KEY, dc.get(DublinCore.PROPERTY_DESCRIPTION));
addMultiValuedFieldToSolrDocument(doc, SolrFields.LANGUAGE_KEY, dc.get(DublinCore.PROPERTY_LANGUAGE));
addMultiValuedFieldToSolrDocument(doc, SolrFields.RIGHTS_HOLDER_KEY, dc.get(DublinCore.PROPERTY_RIGHTS_HOLDER));
addMultiValuedFieldToSolrDocument(doc, SolrFields.SPATIAL_KEY, dc.get(DublinCore.PROPERTY_SPATIAL));
addMultiValuedFieldToSolrDocument(doc, SolrFields.IS_PART_OF_KEY, dc.get(DublinCore.PROPERTY_IS_PART_OF));
addMultiValuedFieldToSolrDocument(doc, SolrFields.REPLACES_KEY, dc.get(DublinCore.PROPERTY_REPLACES));
addMultiValuedFieldToSolrDocument(doc, SolrFields.TYPE_KEY, dc.get(DublinCore.PROPERTY_TYPE));
addMultiValuedFieldToSolrDocument(doc, SolrFields.ACCESS_RIGHTS_KEY, dc.get(DublinCore.PROPERTY_ACCESS_RIGHTS));
addMultiValuedFieldToSolrDocument(doc, SolrFields.LICENSE_KEY, dc.get(DublinCore.PROPERTY_LICENSE));
// add timestamp
doc.setField(SolrFields.LAST_MODIFIED, new Date());
return doc;
}
/**
* Add field to solr document that can contain multiple values. For sorting field, those values are concatenated and
* multivalued field delimiter is used.
*
* @param doc
* {@link SolrInputDocument} for fields to be added to
* @param solrField
* name of the solr field to add value. For sorting field "_sort" is appended
* @param dcValues
* List of Dublin core values to be added to solr document
*/
private void addMultiValuedFieldToSolrDocument(SolrInputDocument doc, String solrField, List<DublinCoreValue> dcValues) {
if (!dcValues.isEmpty()) {
List<String> values = new LinkedList<String>();
StringBuilder builder = new StringBuilder();
values.add(dcValues.get(0).getValue());
builder.append(dcValues.get(0).getValue());
for (int i = 1; i < dcValues.size(); i++) {
values.add(dcValues.get(i).getValue());
builder.append(SOLR_MULTIVALUED_DELIMETER);
builder.append(dcValues.get(i).getValue());
}
doc.addField(solrField, values);
doc.setField(solrField + "_sort", builder.toString());
}
}
/*
* (non-Javadoc)
*
* @see org.opencastproject.scheduler.impl.SchedulerServiceIndex#count()
*/
@Override
public long count() throws SchedulerServiceDatabaseException {
try {
QueryResponse response = solrServer.query(new SolrQuery("*:*"));
return response.getResults().getNumFound();
} catch (SolrServerException e) {
throw new SchedulerServiceDatabaseException(e);
}
}
/**
* Retrieves SolrDocument by specified ID. If such document does not exist, null is returned.
*
* @param id
* ID of document to be retrieved
* @return matching SolrDocument or null
* @throws SchedulerServiceDatabaseException
* if exception occurred
*/
private SolrDocument retrieveDocumentById(long id) throws SchedulerServiceDatabaseException {
String solrQueryString = SolrFields.ID_KEY + ":" + ClientUtils.escapeQueryChars(Long.toString(id));
SolrQuery q = new SolrQuery(solrQueryString);
QueryResponse response;
try {
response = solrServer.query(q);
} catch (SolrServerException e) {
logger.error("Could not perform event retrieval: {}", e);
throw new SchedulerServiceDatabaseException(e);
}
return response.getResults().isEmpty() ? null : response.getResults().get(0);
}
/**
* Appends query parameters to a solr query
*
* @param sb
* The {@link StringBuilder} containing the query
* @param key
* the key for this search parameter
* @param value
* the value for this search parameter
* @return the appended {@link StringBuilder}
*/
private StringBuilder append(StringBuilder sb, String key, String value) {
if (StringUtils.isBlank(key) || StringUtils.isBlank(value)) {
return sb;
}
if (sb.length() > 0) {
sb.append(" AND ");
}
sb.append(key);
sb.append(":");
sb.append(ClientUtils.escapeQueryChars(value));
return sb;
}
/**
* Appends query parameters to a solr query
*
* @param sb
* The {@link StringBuilder} containing the query
* @param key
* the key for this search parameter
* @param value
* the value for this search parameter
* @return the appended {@link StringBuilder}
*/
private StringBuilder append(StringBuilder sb, String key, boolean value) {
if (sb.length() > 0) {
sb.append(" AND ");
}
sb.append(key);
sb.append(":");
sb.append(value);
return sb;
}
/**
* Appends query parameters to a solr query in a way that they are found even though they are not treated as a full
* word in solr.
*
* @param sb
* The {@link StringBuilder} containing the query
* @param key
* the key for this search parameter
* @param value
* the value for this search parameter
* @return the appended {@link StringBuilder}
*/
private StringBuilder appendFuzzy(StringBuilder sb, String key, String value) {
if (StringUtils.isBlank(key) || StringUtils.isBlank(value)) {
return sb;
}
if (sb.length() > 0) {
sb.append(" AND ");
}
sb.append("(");
sb.append(key).append(":").append(ClientUtils.escapeQueryChars(value));
sb.append(" OR ");
sb.append(key).append(":*").append(ClientUtils.escapeQueryChars(value)).append("*");
sb.append(")");
return sb;
}
/**
* Appends query parameters to a solr query
*
* @param sb
* The {@link StringBuilder} containing the query
* @param key
* the key for this search parameter
* @return the appended {@link StringBuilder}
*/
private StringBuilder append(StringBuilder sb, String key, Date startDate, Date endDate) {
if (StringUtils.isBlank(key) || (startDate == null && endDate == null)) {
return sb;
}
if (sb.length() > 0) {
sb.append(" AND ");
}
if (startDate == null)
startDate = new Date(0);
if (endDate == null)
endDate = new Date(Long.MAX_VALUE);
sb.append(key);
sb.append(":");
sb.append(SolrUtils.serializeDateRange(option(startDate), option(endDate)));
return sb;
}
/**
* Builds a solr search query from a {@link SchedulerQuery}.
*
* @param query
* the series query
* @return the solr query string
*/
protected String buildSolrQueryString(SchedulerQuery query) {
StringBuilder sb = new StringBuilder();
append(sb, SolrFields.ID_KEY, query.getIdentifier());
append(sb, SolrFields.IS_PART_OF_KEY, query.getSeriesId());
appendFuzzy(sb, SolrFields.TITLE_KEY, query.getTitle());
appendFuzzy(sb, SolrFields.FULLTEXT_KEY, query.getText());
appendFuzzy(sb, SolrFields.CREATOR_KEY, query.getCreator());
appendFuzzy(sb, SolrFields.CONTRIBUTOR_KEY, query.getContributor());
append(sb, SolrFields.LANGUAGE_KEY, query.getLanguage());
append(sb, SolrFields.LICENSE_KEY, query.getLicense());
appendFuzzy(sb, SolrFields.SUBJECT_KEY, query.getSubject());
appendFuzzy(sb, SolrFields.ABSTRACT_KEY, query.getAbstract());
appendFuzzy(sb, SolrFields.DESCRIPTION_KEY, query.getDescription());
appendFuzzy(sb, SolrFields.PUBLISHER_KEY, query.getPublisher());
append(sb, SolrFields.SPATIAL_KEY, query.getSpatial());
appendFuzzy(sb, SolrFields.RIGHTS_HOLDER_KEY, query.getRightsHolder());
appendFuzzy(sb, SolrFields.SUBJECT_KEY, query.getSubject());
append(sb, SolrFields.CREATED_KEY, query.getCreatedFrom(), query.getCreatedTo());
append(sb, SolrFields.STARTS_KEY, query.getStartsFrom(), query.getStartsTo());
append(sb, SolrFields.ENDS_KEY, query.getEndsFrom(), query.getEndsTo());
append(sb, SolrFields.OPT_OUT, query.isOptOut());
append(sb, SolrFields.BLACKLISTED, query.isBlacklisted());
if (query.getIdsList() != null) {
if (sb.length() > 0) {
sb.append(" AND ");
}
sb.append("(");
List<Long> ids = query.getIdsList();
for (int i = 0; i < ids.size(); i++) {
String id = Long.toString(ids.get(i));
if (StringUtils.isNotEmpty(id)) {
sb.append(SolrFields.ID_KEY);
sb.append(":");
sb.append(ClientUtils.escapeQueryChars(id));
}
if (i < ids.size() - 1) {
sb.append(" OR ");
}
}
sb.append(")");
}
// If we're looking for anything, set the query to a wildcard search
if (sb.length() == 0) {
sb.append("*:*");
}
logger.debug("Solr query: " + sb.toString());
return sb.toString();
}
/**
* Returns the search index' field name that corresponds to the sort field.
*
* @param sort
* the sort field
* @return the field name in the search index
*/
protected String getSortField(SchedulerQuery.Sort sort) {
switch (sort) {
case ABSTRACT:
return SolrFields.ABSTRACT_KEY;
case ACCESS:
return SolrFields.ACCESS_RIGHTS_KEY;
case AVAILABLE_FROM:
return SolrFields.AVAILABLE_FROM_KEY;
case AVAILABLE_TO:
return SolrFields.AVAILABLE_TO_KEY;
case CONTRIBUTOR:
return SolrFields.CONTRIBUTOR_KEY;
case CREATED:
return SolrFields.CREATED_KEY;
case CREATOR:
return SolrFields.CREATOR_KEY;
case DESCRIPTION:
return SolrFields.DESCRIPTION_KEY;
case IS_PART_OF:
return SolrFields.IS_PART_OF_KEY;
case LANGUAGE:
return SolrFields.LANGUAGE_KEY;
case LICENCE:
return SolrFields.LICENSE_KEY;
case PUBLISHER:
return SolrFields.PUBLISHER_KEY;
case REPLACES:
return SolrFields.REPLACES_KEY;
case RIGHTS_HOLDER:
return SolrFields.RIGHTS_HOLDER_KEY;
case SPATIAL:
return SolrFields.SPATIAL_KEY;
case SUBJECT:
return SolrFields.SUBJECT_KEY;
case TITLE:
return SolrFields.TITLE_KEY;
case TYPE:
return SolrFields.TYPE_KEY;
case EVENT_START:
return SolrFields.STARTS_KEY;
default:
throw new IllegalArgumentException("No mapping found between sort field and index");
}
}
@Override
public DublinCoreCatalogList search(SchedulerQuery query) throws SchedulerServiceDatabaseException {
List<DublinCoreCatalog> resultList;
try {
QueryResponse response = solrServer.query(getSolrQuery(query));
SolrDocumentList items = response.getResults();
resultList = new LinkedList<DublinCoreCatalog>();
// Iterate through the results
for (SolrDocument doc : items) {
DublinCoreCatalog dc = parseDublinCore((String) doc.get(SolrFields.XML_KEY));
resultList.add(dc);
}
return new DublinCoreCatalogList(resultList, response.getResults().getNumFound());
} catch (Exception e) {
throw new SchedulerServiceDatabaseException(e);
}
}
@Override
public List<Tuple<String, String>> calendarSearch(SchedulerQuery query) throws SchedulerServiceDatabaseException {
List<Tuple<String, String>> resultList = new LinkedList<Tuple<String, String>>();
try {
QueryResponse response = solrServer.query(getSolrQuery(query));
SolrDocumentList items = response.getResults();
// Iterate through the results
for (SolrDocument doc : items) {
String dc = (String) doc.get(SolrFields.XML_KEY);
String serializedCA = (String) doc.get(SolrFields.CA_PROPERTIES);
resultList.add(Tuple.tuple(dc, serializedCA));
}
return resultList;
} catch (Exception e) {
throw new SchedulerServiceDatabaseException(e);
}
}
private SolrQuery getSolrQuery(SchedulerQuery query) {
SolrQuery solrQuery = new SolrQuery();
if (query == null) {
query = new SchedulerQuery();
}
String solrQueryString = buildSolrQueryString(query);
solrQuery.setQuery(solrQueryString);
solrQuery.setRows(Integer.MAX_VALUE);
if (query.getSort() != null) {
SolrQuery.ORDER order = query.isSortAscending() ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
solrQuery.addSortField(getSortField(query.getSort()) + "_sort", order);
}
if (!SchedulerQuery.Sort.EVENT_START.equals(query.getSort())) {
solrQuery.addSortField(getSortField(SchedulerQuery.Sort.EVENT_START) + "_sort", SolrQuery.ORDER.asc);
}
return solrQuery;
}
@Override
public void delete(final long id) throws SchedulerServiceDatabaseException {
if (synchronousIndexing) {
try {
synchronized (solrServer) {
DublinCoreCatalog catalog = getDublinCore(id);
solrServer.deleteById(Long.toString(id));
solrServer.commit();
if (catalog != null) {
String spatial = catalog.getFirst(DublinCoreCatalog.PROPERTY_SPATIAL);
if (StringUtils.isNotBlank(spatial)) {
logger.debug("Marking calendar feed for {} as modified", spatial);
touchLastEntry(spatial);
}
}
}
} catch (Exception e) {
throw new SchedulerServiceDatabaseException(e);
}
} else {
indexingExecutor.submit(new Runnable() {
@Override
public void run() {
try {
synchronized (solrServer) {
DublinCoreCatalog catalog = getDublinCore(id);
solrServer.deleteById(Long.toString(id));
solrServer.commit();
if (catalog != null) {
String spatial = catalog.getFirst(DublinCoreCatalog.PROPERTY_SPATIAL);
if (StringUtils.isNotBlank(spatial)) {
logger.debug("Marking calendar feed for {} as modified", spatial);
touchLastEntry(spatial);
}
}
}
} catch (Exception e) {
logger.warn("Could not delete from index event {}: {}", id, e.getMessage());
}
}
});
}
}
@Override
public DublinCoreCatalog getDublinCore(long eventId) throws SchedulerServiceDatabaseException, NotFoundException {
SolrDocument result = retrieveDocumentById(eventId);
if (result == null) {
logger.info("No event exists with ID {}", eventId);
throw new NotFoundException("Event with ID " + eventId + " does not exist");
} else {
String dcXML = (String) result.get(SolrFields.XML_KEY);
DublinCoreCatalog dc;
try {
dc = parseDublinCore(dcXML);
} catch (IOException e) {
logger.error("Could not parse Dublin core: {}", e);
throw new SchedulerServiceDatabaseException(e);
}
return dc;
}
}
@Override
public Properties getCaptureAgentProperties(long eventId) throws SchedulerServiceDatabaseException, NotFoundException {
SolrDocument result = retrieveDocumentById(eventId);
if (result == null) {
logger.info("No event exists with ID {}", eventId);
throw new NotFoundException("Event with ID " + eventId + " does not exist");
} else {
String serializedCA = (String) result.get(SolrFields.CA_PROPERTIES);
Properties caProperties = null;
if (serializedCA != null) {
try {
caProperties = parseProperties(serializedCA);
} catch (IOException e) {
logger.error("Could not parse capture agent properties: {}", e);
throw new SchedulerServiceDatabaseException(e);
}
}
return caProperties;
}
}
@Override
public boolean isOptOut(long eventId) throws NotFoundException, SchedulerServiceDatabaseException {
SolrDocument result = retrieveDocumentById(eventId);
if (result == null) {
logger.info("No event exists with ID {}", eventId);
throw new NotFoundException("Event with ID " + eventId + " does not exist");
}
return BooleanUtils.toBoolean((Boolean) result.get(SolrFields.OPT_OUT));
}
@Override
public boolean isBlacklisted(long eventId) throws NotFoundException, SchedulerServiceDatabaseException {
SolrDocument result = retrieveDocumentById(eventId);
if (result == null) {
logger.info("No event exists with ID {}", eventId);
throw new NotFoundException("Event with ID " + eventId + " does not exist");
}
return BooleanUtils.toBoolean((Boolean) result.get(SolrFields.BLACKLISTED));
}
/**
* Touches the most recent entry by updating its last modification date.
*
* @param spatial
* the spatial parameter aka capture agent identifier
* @throws SchedulerServiceDatabaseException
* if updating of the last modified value fails
*/
private void touchLastEntry(String spatial) throws SchedulerServiceDatabaseException {
SchedulerQuery filter = new SchedulerQuery().setSpatial(spatial);
SolrQuery q = new SolrQuery(buildSolrQueryString(filter));
q.addSortField(SolrFields.LAST_MODIFIED + "_sort", SolrQuery.ORDER.desc);
q.setRows(1);
QueryResponse response = null;
// See if there is a matching entry
try {
response = solrServer.query(q);
if (response.getResults().isEmpty()) {
logger.debug("No remaining events scheduled for {}", spatial);
return;
}
} catch (SolrServerException e) {
logger.error("Could not complete query request: {}", e);
throw new SchedulerServiceDatabaseException(e);
}
// Update that entry and write it back
SolrDocument doc = response.getResults().get(0);
final SolrInputDocument inputDoc = ClientUtils.toSolrInputDocument(doc);
inputDoc.setField(SolrFields.LAST_MODIFIED, new Date());
try {
solrServer.add(inputDoc);
solrServer.commit();
} catch (SolrServerException e) {
logger.error("Error updating scheduler entry: {}", e);
throw new SchedulerServiceDatabaseException(e);
} catch (IOException e) {
logger.error("Error updating scheduler entry: {}", e);
throw new SchedulerServiceDatabaseException(e);
}
}
@Override
public Map<String, Date> getLastModifiedDate(SchedulerQuery filter) throws SchedulerServiceDatabaseException {
QueryResponse response;
try {
String solrQueryString = buildSolrQueryString(filter);
SolrQuery q = new SolrQuery(solrQueryString);
q.addSortField(SolrFields.LAST_MODIFIED + "_sort", SolrQuery.ORDER.desc);
q.setRows(Integer.MAX_VALUE);
response = solrServer.query(q);
} catch (SolrServerException e) {
logger.error("Could not complete query request: {}", e);
throw new SchedulerServiceDatabaseException(e);
}
if (response.getResults().isEmpty()) {
logger.debug("No events scheduled for {}", filter);
return new HashMap<String, Date>();
}
try {
// Iterate through the results
Map<String, Date> lastModifiedDates = new HashMap<String, Date>();
for (SolrDocument doc : response.getResults()) {
String agentId = (String) doc.get(SolrFields.SPATIAL_KEY);
if (agentId == null || lastModifiedDates.containsKey(agentId))
continue;
Date date = (Date) doc.get(SolrFields.LAST_MODIFIED);
lastModifiedDates.put(agentId, date);
}
return lastModifiedDates;
} catch (Exception e) {
throw new SchedulerServiceDatabaseException(e);
}
}
/**
* Clears the index of all series instances.
*/
public void clear() throws SchedulerServiceDatabaseException {
if (synchronousIndexing) {
try {
synchronized (solrServer) {
solrServer.deleteByQuery("*:*");
solrServer.commit();
}
} catch (Exception e) {
throw new SchedulerServiceDatabaseException(e);
}
} else {
indexingExecutor.submit(new Runnable() {
@Override
public void run() {
try {
synchronized (solrServer) {
solrServer.deleteByQuery("*:*");
solrServer.commit();
}
} catch (Exception e) {
logger.warn("Could not clear index: {}", e.getMessage());
}
}
});
}
}
/**
* Serializes Dublin core and returns serialized string.
*
* @param dc
* {@link DublinCoreCatalog} to be serialized
* @return String representation of serialized Dublin core
* @throws IOException
* if serialization fails
*/
private String serializeDublinCore(DublinCoreCatalog dc) throws IOException {
InputStream in = dcService.serialize(dc);
StringWriter writer = new StringWriter();
IOUtils.copy(in, writer, "UTF-8");
return writer.toString();
}
/**
* Parses Dublin core stored as string.
*
* @param dcXML
* string representation of Dublin core
* @return parsed {@link DublinCoreCatalog}
* @throws IOException
* if parsing fails
*/
private DublinCoreCatalog parseDublinCore(String dcXML) throws IOException {
DublinCoreCatalog dc = dcService.load(new ByteArrayInputStream(dcXML.getBytes("UTF-8")));
return dc;
}
/**
* Serializes Properties to String.
*
* @param caProperties
* Properties to be serialized
* @return serialized properties
* @throws IOException
* if serialization fails
*/
private String serializeProperties(Properties caProperties) throws IOException {
StringWriter writer = new StringWriter();
caProperties.store(writer, "Capture Agent specific data");
return writer.toString();
}
/**
* Parses Properties represented as String.
*
* @param serializedProperties
* properties to be parsed.
* @return parsed properties
* @throws IOException
* if parsing fails
*/
private Properties parseProperties(String serializedProperties) throws IOException {
Properties caProperties = new Properties();
caProperties.load(new StringReader(serializedProperties));
return caProperties;
}
}