/*
* eXist Open Source Native XML Database
* Copyright (C) 2001-07 The eXist Project
* http://exist-db.org
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
* $Id$
*/
package org.exist.storage;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Properties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.exist.EXistException;
import org.exist.backup.ConsistencyCheck;
import org.exist.backup.ErrorReport;
import org.exist.backup.SystemExport;
import org.exist.management.Agent;
import org.exist.management.AgentFactory;
import org.exist.management.TaskStatus;
import org.exist.security.PermissionDeniedException;
import org.exist.util.Configuration;
import org.exist.xquery.TerminatedException;
import static java.nio.charset.StandardCharsets.UTF_8;
public class ConsistencyCheckTask implements SystemTask {
private final static Logger LOG = LogManager.getLogger(ConsistencyCheckTask.class);
private String exportDir;
private boolean createBackup = false;
private boolean createZip = true;
private boolean paused = false;
private boolean incremental = false;
private boolean incrementalCheck = false;
private boolean checkDocs = false;
private int maxInc = -1;
private Path lastExportedBackup = null;
private ProcessMonitor.Monitor monitor = new ProcessMonitor.Monitor();
public final static String OUTPUT_PROP_NAME = "output";
public final static String ZIP_PROP_NAME = "zip";
public final static String BACKUP_PROP_NAME = "backup";
public final static String INCREMENTAL_PROP_NAME = "incremental";
public final static String INCREMENTAL_CHECK_PROP_NAME = "incremental-check";
public final static String MAX_PROP_NAME = "max";
public final static String CHECK_DOCS_PROP_NAME = "check-documents";
private final static LoggingCallback logCallback = new LoggingCallback();
@Override
public boolean afterCheckpoint() {
return false;
}
@Override
public String getName() {
return "Consistency Check Task";
}
@Override
public void configure(final Configuration config, final Properties properties) throws EXistException {
exportDir = properties.getProperty(OUTPUT_PROP_NAME, "export");
Path dir = Paths.get(exportDir);
if (!dir.isAbsolute()) {
dir = ((Path) config.getProperty(BrokerPool.PROPERTY_DATA_DIR)).resolve(exportDir);
}
try {
Files.createDirectories(dir);
} catch(final IOException ioe) {
throw new EXistException("Unable to create export directory: " + exportDir, ioe);
}
exportDir = dir.toAbsolutePath().toString();
if (LOG.isDebugEnabled()) {
LOG.debug("Using output directory " + exportDir);
}
final String backup = properties.getProperty(BACKUP_PROP_NAME, "no");
createBackup = backup.equalsIgnoreCase("YES");
final String zip = properties.getProperty(ZIP_PROP_NAME, "yes");
createZip = zip.equalsIgnoreCase("YES");
final String inc = properties.getProperty(INCREMENTAL_PROP_NAME, "no");
incremental = inc.equalsIgnoreCase("YES");
final String incCheck = properties.getProperty(INCREMENTAL_CHECK_PROP_NAME, "yes");
incrementalCheck = incCheck.equalsIgnoreCase("YES");
final String max = properties.getProperty(MAX_PROP_NAME, "5");
try {
maxInc = Integer.parseInt(max);
} catch (final NumberFormatException e) {
throw new EXistException("Parameter 'max' has to be an integer");
}
final String check = properties.getProperty(CHECK_DOCS_PROP_NAME, "no");
checkDocs = check.equalsIgnoreCase("YES");
}
@Override
public void execute(final DBBroker broker) throws EXistException {
final Agent agentInstance = AgentFactory.getInstance();
final BrokerPool brokerPool = broker.getBrokerPool();
final TaskStatus endStatus = new TaskStatus(TaskStatus.Status.STOPPED_OK);
agentInstance.changeStatus(brokerPool, new TaskStatus(TaskStatus.Status.INIT));
if (paused) {
LOG.info("Consistency check is paused.");
agentInstance.changeStatus(brokerPool, new TaskStatus(TaskStatus.Status.PAUSED));
return;
}
brokerPool.getProcessMonitor().startJob(ProcessMonitor.ACTION_BACKUP, null, monitor);
PrintWriter report = null;
try {
boolean doBackup = createBackup;
// TODO: don't use the direct access feature for now. needs more testing
List<ErrorReport> errors = null;
if (!incremental || incrementalCheck) {
LOG.info("Starting consistency check...");
report = openLog();
final CheckCallback cb = new CheckCallback(report);
final ConsistencyCheck check = new ConsistencyCheck(broker, false, checkDocs);
agentInstance.changeStatus(brokerPool, new TaskStatus(TaskStatus.Status.RUNNING_CHECK));
errors = check.checkAll(cb);
if (!errors.isEmpty()) {
endStatus.setStatus(TaskStatus.Status.STOPPED_ERROR);
endStatus.setReason(errors);
LOG.error("Errors found: " + errors.size());
doBackup = true;
if (fatalErrorsFound(errors)) {
LOG.error("Fatal errors were found: pausing the consistency check task.");
paused = true;
}
}
LOG.info("Finished consistency check");
}
if (doBackup) {
LOG.info("Starting backup...");
final SystemExport sysexport = new SystemExport(broker, logCallback, monitor, false);
lastExportedBackup = sysexport.export(exportDir, incremental, maxInc, createZip, errors);
agentInstance.changeStatus(brokerPool, new TaskStatus(TaskStatus.Status.RUNNING_BACKUP));
if (lastExportedBackup != null) {
LOG.info("Created backup to file: " + lastExportedBackup.toAbsolutePath().toString());
}
LOG.info("Finished backup");
}
} catch (final TerminatedException | PermissionDeniedException e) {
throw new EXistException(e.getMessage(), e);
} finally {
if (report != null) {
report.close();
}
agentInstance.changeStatus(brokerPool, endStatus);
brokerPool.getProcessMonitor().endJob();
}
}
/**
* Gets the last exported backup
*/
public Path getLastExportedBackup()
{
return lastExportedBackup;
}
private boolean fatalErrorsFound(final List<ErrorReport> errors) {
for (final ErrorReport error : errors) {
switch (error.getErrcode()) {
// the following errors are considered fatal: export the db and
// stop the task
case ErrorReport.CHILD_COLLECTION:
case ErrorReport.RESOURCE_ACCESS_FAILED:
return true;
}
}
// no fatal errors
return false;
}
private PrintWriter openLog() throws EXistException {
try {
final Path file = SystemExport.getUniqueFile("report", ".log", exportDir);
return new PrintWriter(Files.newBufferedWriter(file, UTF_8));
} catch (final IOException e) {
throw new EXistException("ERROR: failed to create report file in " + exportDir, e);
}
}
private static class LoggingCallback implements SystemExport.StatusCallback {
@Override
public void startCollection(final String path) throws TerminatedException {
}
@Override
public void startDocument(final String name, final int current, final int count) throws TerminatedException {
}
@Override
public void error(final String message, final Throwable exception) {
LOG.error(message, exception);
}
}
private class CheckCallback implements ConsistencyCheck.ProgressCallback, SystemExport.StatusCallback {
private final PrintWriter log;
private boolean errorFound = false;
private CheckCallback(final PrintWriter log) {
this.log = log;
}
@Override
public void startDocument(final String name, final int current, final int count) throws TerminatedException {
if (!monitor.proceed()) {
throw new TerminatedException("consistency check terminated");
}
if ((current % 1000 == 0) || (current == count)) {
log.write(" DOCUMENT: ");
log.write(Integer.toString(current));
log.write(" of ");
log.write(Integer.valueOf(count).toString());
log.write('\n');
log.flush();
}
}
@Override
public void startCollection(final String path) throws TerminatedException {
if (!monitor.proceed()) {
throw new TerminatedException("consistency check terminated");
}
if (errorFound) {
log.write("----------------------------------------------\n");
}
errorFound = false;
log.write("COLLECTION: ");
log.write(path);
log.write('\n');
log.flush();
}
@Override
public void error(final ErrorReport error) {
log.write("----------------------------------------------\n");
log.write(error.toString());
log.write('\n');
log.flush();
}
@Override
public void error(final String message, final Throwable exception) {
log.write("----------------------------------------------\n");
log.write("EXPORT ERROR: ");
log.write(message);
log.write('\n');
exception.printStackTrace(log);
log.flush();
}
}
}