/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.util;
import org.apache.commons.cli.*;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.LukeResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.common.luke.FieldFlag;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.FacetParams;
import org.dspace.core.ConfigurationManager;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.net.URL;
import java.text.*;
import java.util.*;
/**
* Utility class to export, clear and import Solr indexes.
* @author Andrea Schweer schweer@waikato.ac.nz for the LCoNZ Institutional Research Repositories
*/
public class SolrImportExport
{
private static final ThreadLocal<DateFormat> SOLR_DATE_FORMAT;
private static final ThreadLocal<DateFormat> SOLR_DATE_FORMAT_NO_MS;
private static final ThreadLocal<DateFormat> EXPORT_DATE_FORMAT;
private static final String EXPORT_SEP = "_export_";
static
{
SOLR_DATE_FORMAT = new ThreadLocal<DateFormat>() {
@Override
protected DateFormat initialValue() {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
return simpleDateFormat;
}
};
SOLR_DATE_FORMAT_NO_MS = new ThreadLocal<DateFormat>() {
@Override
protected DateFormat initialValue() {
return new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
}
};
EXPORT_DATE_FORMAT = new ThreadLocal<DateFormat>() {
@Override
protected DateFormat initialValue() {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM");
simpleDateFormat.setTimeZone(TimeZone.getDefault());
return simpleDateFormat;
}
};
}
private static final String ACTION_OPTION = "a";
private static final String CLEAR_OPTION = "c";
private static final String OVERWRITE_OPTION = "f";
private static final String DIRECTORY_OPTION = "d";
private static final String HELP_OPTION = "h";
private static final String INDEX_NAME_OPTION = "i";
private static final String KEEP_OPTION = "k";
private static final String LAST_OPTION = "l";
public static final int ROWS_PER_FILE = 10_000;
private static final String MULTIPLE_VALUES_SPLITTER = ",";
private static final Logger log = Logger.getLogger(SolrImportExport.class);
/**
* Entry point for command-line invocation
* @param args command-line arguments; see help for description
* @throws ParseException if the command-line arguments cannot be parsed
*/
public static void main(String[] args) throws ParseException
{
CommandLineParser parser = new PosixParser();
Options options = makeOptions();
try
{
CommandLine line = parser.parse(options, args);
if (line.hasOption(HELP_OPTION))
{
printHelpAndExit(options, 0);
}
String[] indexNames = {"statistics"};
if (line.hasOption(INDEX_NAME_OPTION))
{
indexNames = line.getOptionValues(INDEX_NAME_OPTION);
}
else
{
System.err.println("No index name provided, defaulting to \"statistics\".");
}
String directoryName = makeDirectoryName(line.getOptionValue(DIRECTORY_OPTION));
String action = line.getOptionValue(ACTION_OPTION, "export");
if ("import".equals(action))
{
for (String indexName : indexNames)
{
File importDir = new File(directoryName);
if (!importDir.exists() || !importDir.canRead())
{
System.err.println("Import directory " + directoryName
+ " doesn't exist or is not readable by the current user. Not importing index "
+ indexName);
continue; // skip this index
}
try
{
String solrUrl = makeSolrUrl(indexName);
boolean clear = line.hasOption(CLEAR_OPTION);
importIndex(indexName, importDir, solrUrl, clear);
}
catch (IOException | SolrServerException | SolrImportExportException e)
{
System.err.println("Problem encountered while trying to import index " + indexName + ".");
e.printStackTrace(System.err);
}
}
}
else if ("export".equals(action))
{
for (String indexName : indexNames)
{
String lastValue = line.getOptionValue(LAST_OPTION);
File exportDir = new File(directoryName);
if (exportDir.exists() && !exportDir.canWrite())
{
System.err.println("Export directory " + directoryName
+ " is not writable by the current user. Not exporting index "
+ indexName);
continue;
}
if (!exportDir.exists())
{
boolean created = exportDir.mkdirs();
if (!created)
{
System.err.println("Export directory " + directoryName
+ " could not be created. Not exporting index " + indexName);
}
continue;
}
try
{
String solrUrl = makeSolrUrl(indexName);
String timeField = makeTimeField(indexName);
exportIndex(indexName, exportDir, solrUrl, timeField, lastValue, line.hasOption(OVERWRITE_OPTION));
}
catch (SolrServerException | IOException | SolrImportExportException e)
{
System.err.println("Problem encountered while trying to export index " + indexName + ".");
e.printStackTrace(System.err);
}
}
}
else if ("reindex".equals(action))
{
for (String indexName : indexNames)
{
try {
boolean keepExport = line.hasOption(KEEP_OPTION);
boolean overwrite = line.hasOption(OVERWRITE_OPTION);
reindex(indexName, directoryName, keepExport, overwrite);
} catch (IOException | SolrServerException | SolrImportExportException e) {
e.printStackTrace();
}
}
}
else
{
System.err.println("Unknown action " + action + "; must be import, export or reindex.");
printHelpAndExit(options, 1);
}
}
catch (ParseException e)
{
System.err.println("Cannot read command options");
printHelpAndExit(options, 1);
}
}
private static Options makeOptions() {
Options options = new Options();
options.addOption(ACTION_OPTION, "action", true, "The action to perform: import, export or reindex. Default: export.");
options.addOption(CLEAR_OPTION, "clear", false, "When importing, also clear the index first. Ignored when action is export or reindex.");
options.addOption(OVERWRITE_OPTION, "force-overwrite", false, "When exporting or re-indexing, allow overwrite of existing export files");
options.addOption(DIRECTORY_OPTION, "directory", true,
"The absolute path for the directory to use for import or export. If omitted, [dspace]/solr-export is used.");
options.addOption(HELP_OPTION, "help", false, "Get help on options for this command.");
options.addOption(INDEX_NAME_OPTION, "index-name", true,
"The names of the indexes to process. At least one is required. Available indexes are: authority, statistics.");
options.addOption(KEEP_OPTION, "keep", false, "When reindexing, keep the contents of the data export directory." +
" By default, the contents of this directory will be deleted once the reindex has finished." +
" Ignored when action is export or import.");
options.addOption(LAST_OPTION, "last", true, "When exporting, export records from the last [timeperiod] only." +
" This can be one of: 'd' (beginning of yesterday through to now);" +
" 'm' (beginning of the previous month through to end of the previous month);" +
" a number, in which case the last [number] of days are exported, through to now (use 0 for today's data)." +
" Date calculation is done in UTC. If omitted, all documents are exported.");
return options;
}
/**
* Reindexes the specified core
*
* @param indexName the name of the core to reindex
* @param exportDirName the name of the directory to use for export. If this directory doesn't exist, it will be created.
* @param keepExport whether to keep the contents of the exportDir after the reindex. If keepExport is false and the
* export directory was created by this method, the export directory will be deleted at the end of the reimport.
* @param overwrite allow export files to be overwritten during re-index
*/
private static void reindex(String indexName, String exportDirName, boolean keepExport, boolean overwrite)
throws IOException, SolrServerException, SolrImportExportException {
String tempIndexName = indexName + "-temp";
String origSolrUrl = makeSolrUrl(indexName);
String baseSolrUrl = StringUtils.substringBeforeLast(origSolrUrl, "/"); // need to get non-core solr URL
String tempSolrUrl = baseSolrUrl + "/" + tempIndexName;
//The configuration details for the statistics shards reside within the "statistics" folder
String instanceIndexName = indexName.startsWith("statistics-") ? "statistics" : indexName;
String solrInstanceDir = ConfigurationManager.getProperty("dspace.dir") + File.separator + "solr" + File.separator + instanceIndexName;
// the [dspace]/solr/[indexName]/conf directory needs to be available on the local machine for this to work
// -- we need access to the schema.xml and solrconfig.xml file, plus files referenced from there
// if this directory can't be found, output an error message and skip this index
File solrInstance = new File(solrInstanceDir);
if (!solrInstance.exists() || !solrInstance.canRead() || !solrInstance.isDirectory())
{
throw new SolrImportExportException("Directory " + solrInstanceDir + "/conf/ doesn't exist or isn't readable." +
" The reindexing process requires the Solr configuration directory for this index to be present on the local machine" +
" even if Solr is running on a different host. Not reindexing index " + indexName);
}
String timeField = makeTimeField(indexName);
// Ensure the export directory exists and is writable
File exportDir = new File(exportDirName);
boolean createdExportDir = exportDir.mkdirs();
if (!createdExportDir && !exportDir.exists())
{
throw new SolrImportExportException("Could not create export directory " + exportDirName);
}
if (!exportDir.canWrite())
{
throw new SolrImportExportException("Can't write to export directory " + exportDirName);
}
try
{
HttpSolrServer adminSolr = new HttpSolrServer(baseSolrUrl);
// try to find out size of core and compare with free space in export directory
CoreAdminResponse status = CoreAdminRequest.getStatus(indexName, adminSolr);
Object coreSizeObj = status.getCoreStatus(indexName).get("sizeInBytes");
long coreSize = coreSizeObj != null ? Long.valueOf(coreSizeObj.toString()) : -1;
long usableExportSpace = exportDir.getUsableSpace();
if (coreSize >= 0 && usableExportSpace < coreSize)
{
System.err.println("Not enough space in export directory " + exportDirName
+ "; need at least as much space as the index ("
+ FileUtils.byteCountToDisplaySize(coreSize)
+ ") but usable space in export directory is only "
+ FileUtils.byteCountToDisplaySize(usableExportSpace)
+ ". Not continuing with reindex, please use the " + DIRECTORY_OPTION
+ " option to specify an alternative export directy with sufficient space.");
return;
}
// Create a temp directory to store temporary core data
File tempDataDir = new File(ConfigurationManager.getProperty("dspace.dir") + File.separator + "temp" + File.separator + "solr-data");
boolean createdTempDataDir = tempDataDir.mkdirs();
if (!createdTempDataDir && !tempDataDir.exists())
{
throw new SolrImportExportException("Could not create temporary data directory " + tempDataDir.getCanonicalPath());
}
if (!tempDataDir.canWrite())
{
throw new SolrImportExportException("Can't write to temporary data directory " + tempDataDir.getCanonicalPath());
}
try
{
// create a temporary core to hold documents coming in during the reindex
CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create();
createRequest.setInstanceDir(solrInstanceDir);
createRequest.setDataDir(tempDataDir.getCanonicalPath());
createRequest.setCoreName(tempIndexName);
createRequest.process(adminSolr).getStatus();
}
catch (SolrServerException e)
{
// try to continue -- it may just be that the core already existed from a previous, failed attempt
System.err.println("Caught exception when trying to create temporary core: " + e.getMessage() + "; trying to recover.");
e.printStackTrace(System.err);
}
// swap actual core with temporary one
CoreAdminRequest swapRequest = new CoreAdminRequest();
swapRequest.setCoreName(indexName);
swapRequest.setOtherCoreName(tempIndexName);
swapRequest.setAction(CoreAdminParams.CoreAdminAction.SWAP);
swapRequest.process(adminSolr);
try
{
// export from the actual core (from temp core name, actual data dir)
exportIndex(indexName, exportDir, tempSolrUrl, timeField, overwrite);
// clear actual core (temp core name, clearing actual data dir) & import
importIndex(indexName, exportDir, tempSolrUrl, true);
}
catch (Exception e)
{
// we ran into some problems with the export/import -- keep going to try and restore the solr cores
System.err.println("Encountered problem during reindex: " + e.getMessage() + ", will attempt to restore Solr cores");
e.printStackTrace(System.err);
}
// commit changes
HttpSolrServer origSolr = new HttpSolrServer(origSolrUrl);
origSolr.commit();
// swap back (statistics now going to actual core name in actual data dir)
swapRequest = new CoreAdminRequest();
swapRequest.setCoreName(tempIndexName);
swapRequest.setOtherCoreName(indexName);
swapRequest.setAction(CoreAdminParams.CoreAdminAction.SWAP);
swapRequest.process(adminSolr);
// export all docs from now-temp core into export directory -- this won't cause name collisions with the actual export
// because the core name for the temporary export has -temp in it while the actual core doesn't
exportIndex(tempIndexName, exportDir, tempSolrUrl, timeField, overwrite);
// ...and import them into the now-again-actual core *without* clearing
importIndex(tempIndexName, exportDir, origSolrUrl, false);
// commit changes
origSolr.commit();
// unload now-temp core (temp core name)
CoreAdminRequest.unloadCore(tempIndexName, false, false, adminSolr);
// clean up temporary data dir if this method created it
if (createdTempDataDir && tempDataDir.exists())
{
FileUtils.deleteDirectory(tempDataDir);
}
}
finally
{
// clean up export dir if appropriate
if (!keepExport && createdExportDir && exportDir.exists())
{
FileUtils.deleteDirectory(exportDir);
}
}
}
/**
* Exports all documents in the given index to the specified target directory in batches of #ROWS_PER_FILE.
* See #makeExportFilename for the file names that are generated.
*
* @param indexName The index to export.
* @param toDir The target directory for the export. Will be created if it doesn't exist yet. The directory must be writeable.
* @param solrUrl The solr URL for the index to export. Must not be null.
* @param timeField The time field to use for sorting the export. Must not be null.
* @param overwrite If set, allow export files to be overwritten
* @throws SolrServerException if there is a problem with exporting the index.
* @throws IOException if there is a problem creating the files or communicating with Solr.
* @throws SolrImportExportException if there is a problem in communicating with Solr.
*/
public static void exportIndex(String indexName, File toDir, String solrUrl, String timeField, boolean overwrite)
throws SolrServerException, SolrImportExportException, IOException {
exportIndex(indexName, toDir, solrUrl, timeField, null, overwrite);
}
/**
* Import previously exported documents (or externally created CSV files that have the appropriate structure) into the specified index.
* @param indexName the index to import.
* @param fromDir the source directory. Must exist and be readable.
* The importer will look for files whose name starts with <pre>indexName</pre>
* and ends with .csv (to match what is generated by #makeExportFilename).
* @param solrUrl The solr URL for the index to export. Must not be null.
* @param clear if true, clear the index before importing.
* @param overwrite if true, skip _version_ field on import to disable Solr's optimistic concurrency functionality
* @throws IOException if there is a problem reading the files or communicating with Solr.
* @throws SolrServerException if there is a problem reading the files or communicating with Solr.
* @throws SolrImportExportException if there is a problem communicating with Solr.
*/
public static void importIndex(final String indexName, File fromDir, String solrUrl, boolean clear)
throws IOException, SolrServerException, SolrImportExportException
{
if (StringUtils.isBlank(solrUrl))
{
throw new SolrImportExportException("Could not construct solr URL for index" + indexName + ", aborting export.");
}
if (!fromDir.exists() || !fromDir.canRead())
{
throw new SolrImportExportException("Source directory " + fromDir
+ " doesn't exist or isn't readable, aborting export of index "
+ indexName);
}
HttpSolrServer solr = new HttpSolrServer(solrUrl);
// must get multivalue fields before clearing
List<String> multivaluedFields = getMultiValuedFields(solr);
if (clear)
{
clearIndex(solrUrl);
}
File[] files = fromDir.listFiles(new FilenameFilter()
{
@Override
public boolean accept(File dir, String name)
{
return name.startsWith(indexName + EXPORT_SEP) && name.endsWith(".csv");
}
});
if (files == null || files.length == 0)
{
log.warn("No export files found in directory " + fromDir.getCanonicalPath() + " for index " + indexName);
return;
}
Arrays.sort(files);
for (File file : files)
{
log.info("Importing file " + file.getCanonicalPath());
ContentStreamUpdateRequest contentStreamUpdateRequest = new ContentStreamUpdateRequest("/update/csv");
contentStreamUpdateRequest.setParam("skip", "_version_");
for (String mvField : multivaluedFields) {
contentStreamUpdateRequest.setParam("f." + mvField + ".split", "true");
contentStreamUpdateRequest.setParam("f." + mvField + ".separator", MULTIPLE_VALUES_SPLITTER);
}
contentStreamUpdateRequest.setParam("stream.contentType", "text/csv;charset=utf-8");
contentStreamUpdateRequest.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
contentStreamUpdateRequest.addFile(file, "text/csv;charset=utf-8");
solr.request(contentStreamUpdateRequest);
}
solr.commit(true, true);
}
/**
* Determine the names of all multi-valued fields from the data in the index.
* @param solr the solr server to query.
* @return A list containing all multi-valued fields, or an empty list if none are found / there aren't any.
*/
private static List<String> getMultiValuedFields(HttpSolrServer solr)
{
List<String> result = new ArrayList<>();
try
{
LukeRequest request = new LukeRequest();
// this needs to be a non-schema request, otherwise we'll miss dynamic fields
LukeResponse response = request.process(solr);
Map<String, LukeResponse.FieldInfo> fields = response.getFieldInfo();
for (LukeResponse.FieldInfo info : fields.values())
{
if (info.getSchema().contains(FieldFlag.MULTI_VALUED.getAbbreviation() + ""))
{
result.add(info.getName());
}
}
}
catch (IOException | SolrServerException e)
{
log.fatal("Cannot determine which fields are multi valued: " + e.getMessage(), e);
}
return result;
}
/**
* Remove all documents from the Solr index with the given URL, then commit and optimise the index.
*
* @throws IOException if there is a problem in communicating with Solr.
* @throws SolrServerException if there is a problem in communicating with Solr.
* @param solrUrl URL of the Solr core to clear.
*/
public static void clearIndex(String solrUrl) throws IOException, SolrServerException
{
HttpSolrServer solr = new HttpSolrServer(solrUrl);
solr.deleteByQuery("*:*");
solr.commit();
solr.optimize();
}
/**
* Exports documents from the given index to the specified target directory in batches of #ROWS_PER_FILE, starting at fromWhen (or all documents).
* See #makeExportFilename for the file names that are generated.
*
* @param indexName The index to export.
* @param toDir The target directory for the export. Will be created if it doesn't exist yet. The directory must be writeable.
* @param solrUrl The solr URL for the index to export. Must not be null.
* @param timeField The time field to use for sorting the export. Must not be null.
* @param fromWhen Optionally, from when to export. See options for allowed values. If null or empty, all documents will be exported.
* @param overwrite If set, allow export files to be overwritten
* @throws SolrServerException if there is a problem with exporting the index.
* @throws IOException if there is a problem creating the files or communicating with Solr.
* @throws SolrImportExportException if there is a problem in communicating with Solr.
*/
public static void exportIndex(String indexName, File toDir, String solrUrl, String timeField, String fromWhen, boolean overwrite)
throws SolrServerException, IOException, SolrImportExportException
{
log.info(String.format("Export Index [%s] to [%s] using [%s] Time Field[%s] FromWhen[%s]", indexName, toDir, solrUrl, timeField, fromWhen));
if (StringUtils.isBlank(solrUrl))
{
throw new SolrImportExportException("Could not construct solr URL for index" + indexName + ", aborting export.");
}
if (!toDir.exists() || !toDir.canWrite())
{
throw new SolrImportExportException("Target directory " + toDir
+ " doesn't exist or is not writable, aborting export of index "
+ indexName);
}
HttpSolrServer solr = new HttpSolrServer(solrUrl);
SolrQuery query = new SolrQuery("*:*");
if (StringUtils.isNotBlank(fromWhen))
{
String lastValueFilter = makeFilterQuery(timeField, fromWhen);
if (StringUtils.isNotBlank(lastValueFilter))
{
query.addFilterQuery(lastValueFilter);
}
}
query.setRows(0);
query.setGetFieldStatistics(timeField);
Map<String, FieldStatsInfo> fieldInfo = solr.query(query).getFieldStatsInfo();
if (fieldInfo == null || !fieldInfo.containsKey(timeField)) {
log.warn(String.format("Queried [%s]. No fieldInfo found while exporting index [%s] time field [%s] from [%s]. Export cancelled.",
solrUrl, indexName, timeField, fromWhen));
return;
}
FieldStatsInfo timeFieldInfo = fieldInfo.get(timeField);
if (timeFieldInfo == null || timeFieldInfo.getMin() == null) {
log.warn(String.format("Queried [%s]. No earliest date found while exporting index [%s] time field [%s] from [%s]. Export cancelled.",
solrUrl, indexName, timeField, fromWhen));
return;
}
Date earliestTimestamp = (Date) timeFieldInfo.getMin();
query.setGetFieldStatistics(false);
query.clearSorts();
query.setRows(0);
query.setFacet(true);
query.add(FacetParams.FACET_RANGE, timeField);
query.add(FacetParams.FACET_RANGE_START, SOLR_DATE_FORMAT.get().format(earliestTimestamp) + "/MONTH");
query.add(FacetParams.FACET_RANGE_END, "NOW/MONTH+1MONTH");
query.add(FacetParams.FACET_RANGE_GAP, "+1MONTH");
query.setFacetMinCount(1);
List<RangeFacet.Count> monthFacets = solr.query(query).getFacetRanges().get(0).getCounts();
for (RangeFacet.Count monthFacet : monthFacets) {
Date monthStartDate;
String monthStart = monthFacet.getValue();
try
{
monthStartDate = SOLR_DATE_FORMAT_NO_MS.get().parse(monthStart);
}
catch (java.text.ParseException e)
{
throw new SolrImportExportException("Could not read start of month batch as date: " + monthStart, e);
}
int docsThisMonth = monthFacet.getCount();
SolrQuery monthQuery = new SolrQuery("*:*");
monthQuery.setRows(ROWS_PER_FILE);
monthQuery.set("wt", "csv");
monthQuery.set("fl", "*");
monthQuery.setParam("csv.mv.separator", MULTIPLE_VALUES_SPLITTER);
monthQuery.addFilterQuery(timeField + ":[" +monthStart + " TO " + monthStart + "+1MONTH]");
for (int i = 0; i < docsThisMonth; i+= ROWS_PER_FILE)
{
monthQuery.setStart(i);
URL url = new URL(solrUrl + "/select?" + monthQuery.toString());
File file = new File(toDir.getCanonicalPath(), makeExportFilename(indexName, monthStartDate, docsThisMonth, i));
if (file.createNewFile() || overwrite)
{
FileUtils.copyURLToFile(url, file);
String message = String.format("Solr export to file [%s] complete. Export for Index [%s] Month [%s] Batch [%d] Num Docs [%d]",
file.getCanonicalPath(), indexName, monthStart, i, docsThisMonth);
log.info(message);
}
else if (file.exists())
{
String message = String.format("Solr export file [%s] already exists. Export failed for Index [%s] Month [%s] Batch [%d] Num Docs [%d]",
file.getCanonicalPath(), indexName, monthStart, i, docsThisMonth);
throw new SolrImportExportException(message);
}
else
{
String message = String.format("Cannot create solr export file [%s]. Export failed for Index [%s] Month [%s] Batch [%d] Num Docs [%d]",
file.getCanonicalPath(), indexName, monthStart, i, docsThisMonth); throw new SolrImportExportException(message);
}
}
}
}
/**
* Return a filter query that represents the export date range passed in as lastValue
* @param timeField the time field to use for the date range
* @param lastValue the requested date range, see options for acceptable values
* @return a filter query representing the date range, or null if no suitable date range can be created.
*/
private static String makeFilterQuery(String timeField, String lastValue) {
if ("m".equals(lastValue))
{
// export data from the previous month
return timeField + ":[NOW/MONTH-1MONTH TO NOW/MONTH]";
}
int days;
if ("d".equals(lastValue))
{
days = 1;
}
else
{
// other acceptable value: a number, specifying how many days back to export
days = Integer.valueOf(lastValue); // TODO check value?
}
return timeField + ":[NOW/DAY-" + days + "DAYS TO " + SOLR_DATE_FORMAT.get().format(new Date()) + "]";
}
/**
* Return the specified directory name or fall back to a default value.
*
* @param directoryValue a specific directory name. Optional.
* @return directoryValue if given as a non-blank string. A default directory otherwise.
*/
private static String makeDirectoryName(String directoryValue)
{
if (StringUtils.isNotBlank(directoryValue))
{
return directoryValue;
}
return ConfigurationManager.getProperty("dspace.dir") + File.separator + "solr-export" + File.separator;
}
/**
* Creates a filename for the export batch.
*
* @param indexName The name of the index being exported.
* @param exportStart The start timestamp of the export
* @param totalRecords The total number of records in the export.
* @param index The index of the current batch.
* @return A file name that is appropriate to use for exporting the batch of data described by the parameters.
*/
private static String makeExportFilename(String indexName, Date exportStart, long totalRecords, int index)
{
String exportFileNumber = "";
if (totalRecords > ROWS_PER_FILE) {
exportFileNumber = StringUtils.leftPad("" + (index / ROWS_PER_FILE), (int) Math.ceil(Math.log10(totalRecords / ROWS_PER_FILE)), "0");
}
return indexName
+ EXPORT_SEP
+ EXPORT_DATE_FORMAT.get().format(exportStart)
+ (StringUtils.isNotBlank(exportFileNumber) ? "_" + exportFileNumber : "")
+ ".csv";
}
/**
* Returns the full URL for the specified index name.
*
* @param indexName the index name whose Solr URL is required. If the index name starts with
* "statistics" or is "authority", the Solr base URL will be looked up
* in the corresponding DSpace configuration file. Otherwise, it will fall back to a default.
* @return the full URL to the Solr index, as a String.
*/
private static String makeSolrUrl(String indexName)
{
if (indexName.startsWith("statistics"))
{
// TODO account for year shards properly?
return ConfigurationManager.getProperty("solr-statistics", "server") + indexName.replaceFirst("statistics", "");
}
else if ("authority".equals(indexName))
{
return ConfigurationManager.getProperty("solr.authority.server");
}
return "http://localhost:8080/solr/" + indexName; // TODO better default?
}
/**
* Returns a time field for the specified index name that is suitable for incremental export.
*
* @param indexName the index name whose Solr URL is required.
* @return the name of the time field, or null if no suitable field can be determined.
*/
private static String makeTimeField(String indexName)
{
if (indexName.startsWith("statistics"))
{
return "time";
}
else if ("authority".equals(indexName))
{
return "last_modified_date";
}
return null; // TODO some sort of default?
}
/**
* A utility method to print out all available command-line options and exit given the specified code.
*
* @param options the supported options.
* @param exitCode the exit code to use. The method will call System#exit(int) with the given code.
*/
private static void printHelpAndExit(Options options, int exitCode)
{
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp(SolrImportExport.class.getSimpleName() + "\n", options);
System.out.println("\n\nCommand Defaults");
System.out.println("\tsolr-export-statistics [-a export] [-i statistics]");
System.out.println("\tsolr-import-statistics [-a import] [-i statistics]");
System.out.println("\tsolr-reindex-statistics [-a reindex] [-i statistics]");
System.exit(exitCode);
}
}