/*
* Copyright 2013, CMM, University of Queensland.
*
* This file is part of Paul.
*
* Paul is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Paul is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Paul. If not, see <http://www.gnu.org/licenses/>.
*/
package au.edu.uq.cmm.paul.queue;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import au.edu.uq.cmm.paul.PaulConfiguration;
import au.edu.uq.cmm.paul.PaulException;
/**
* Base class for the queue file manager classes. Note that the vast majority
* of this behavior <i>needs</i> to be common to cope with the fact that the
* queue file management strategy can be reconfigured, and therefore the queue
* could contain any mix of copied or symlinked files.
*
* @author scrawley
*
*/
public abstract class AbstractQueueFileManager implements QueueFileManager {
protected static final int RETRY = 10;
protected final File archiveDirectory;
protected final File captureDirectory;
private Logger log;
public AbstractQueueFileManager(PaulConfiguration config, Logger log) {
this.log = log;
this.archiveDirectory = new File(config.getArchiveDirectory());
checkDirectory(this.archiveDirectory, "archive");
this.captureDirectory = new File(config.getCaptureDirectory());
checkDirectory(this.captureDirectory, "capture");
}
private void checkDirectory(File dir, String tag) {
File testFile = new File(dir, "test.txt");
try (OutputStream os = new FileOutputStream(testFile)) {
os.write("1 2 3\n".getBytes());
} catch (IOException ex) {
throw new PaulException("Problem creating file in " +
tag + " directory", ex);
} finally {
testFile.delete();
}
}
protected File copyFile(File source, File target, String area)
throws QueueFileException {
// TODO - if the time taken to copy files is a problem, we could
// potentially improve this by using NIO or memory mapped files.
long size = source.length();
try (FileInputStream is = new FileInputStream(source);
FileOutputStream os = new FileOutputStream(target)) {
byte[] buffer = new byte[(int) Math.min(size, 8192)];
int nosRead;
long totalRead = 0;
while ((nosRead = is.read(buffer, 0, buffer.length)) > 0) {
os.write(buffer, 0, nosRead);
totalRead += nosRead;
}
// If these happen there is something wrong with our copying, locking
// and / or file settling heuristics.
if (totalRead != size) {
log.error("Copied file size discrepancy - initial file size was " + size +
"bytes but we copied " + totalRead + " bytes");
} else if (size != source.length()) {
log.error("File size changed during copy - initial file size was " + size +
"bytes and current size is " + source.length());
}
log.debug("Copied " + totalRead + " bytes from " + source + " to " + target);
return target;
} catch (IOException ex) {
throw new QueueFileException("Problem while copying file to " + area + " area", ex);
}
}
@Override
public final void enqueueFile(String contents, File target, boolean mayExist)
throws QueueFileException {
if (!mayExist && target.exists()) {
throw new QueueFileException("File " + target + " already exists");
}
try (Writer w = new FileWriter(target)) {
w.write(contents);
w.close();
} catch (IOException ex) {
throw new QueueFileException("Problem while saving to a queue file", ex);
}
}
@Override
public final File generateUniqueFile(String suffix, boolean regrabbing)
throws QueueFileException {
String template = regrabbing ? "regrabbed-%d-%d-%d%s" : "file-%d-%d-%d%s";
long threadId = Thread.currentThread().getId();
for (int i = 0; i < RETRY; i++) {
long now = System.currentTimeMillis();
String name = String.format(template, now, threadId, i, suffix);
File file = new File(captureDirectory, name);
if (!file.exists()) {
return file;
}
}
throw new QueueFileException(
RETRY + " attempts to generate a unique filename failed!");
}
@Override
public final File archiveFile(File file) throws QueueFileException {
switch (getFileStatus(file)) {
case NON_EXISTENT:
throw new QueueFileException("File or symlink " + file + " no longer exists");
case CAPTURED_FILE:
case CAPTURED_SYMLINK:
break;
case BROKEN_CAPTURED_SYMLINK:
throw new QueueFileException("Symlink target for " + file + " no longer exists");
default:
throw new QueueFileException("File or symlink " + file + " is not in the queue");
}
File dest = new File(archiveDirectory, file.getName());
if (dest.exists()) {
throw new QueueFileException("Archived file " + dest + " already exists");
}
if (Files.isSymbolicLink(file.toPath())) {
dest = copyFile(file, dest, "archive");
try {
Files.delete(file.toPath());
} catch (IOException ex) {
throw new QueueFileException("Could not remove symlink " + file);
}
} else {
if (!file.renameTo(dest)) {
throw new QueueFileException("File " + file + " could not be renamed to " + dest);
}
}
log.info("File " + file + " archived as " + dest);
return dest;
}
@Override
public void removeFile(File file) throws QueueFileException {
switch (getFileStatus(file)) {
case NON_EXISTENT:
throw new QueueFileException("File or symlink " + file + " no longer exists");
case CAPTURED_FILE:
case CAPTURED_SYMLINK:
case BROKEN_CAPTURED_SYMLINK:
break;
default:
throw new QueueFileException("File or symlink " + file + " is not in the queue");
}
try {
Files.delete(file.toPath());
log.info("File " + file + " deleted from queue area");
} catch (IOException ex) {
throw new QueueFileException("File or symlink " + file + " could not be deleted from queue area", ex);
}
}
@Override
public FileStatus getFileStatus(File file) {
Path path = file.toPath();
File parent = path.getParent().toFile();
if (Files.exists(path)) {
if (parent.equals(captureDirectory)) {
if (Files.isSymbolicLink(path)) {
return FileStatus.CAPTURED_SYMLINK;
} else {
return FileStatus.CAPTURED_FILE;
}
} else if (parent.equals(archiveDirectory)) {
if (Files.isSymbolicLink(path)) {
return FileStatus.ARCHIVED_SYMLINK;
} else {
return FileStatus.ARCHIVED_FILE;
}
} else {
return FileStatus.NOT_OURS;
}
} else {
if (Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
if (parent.equals(captureDirectory)) {
return FileStatus.BROKEN_CAPTURED_SYMLINK;
} else if (parent.equals(archiveDirectory)) {
return FileStatus.BROKEN_ARCHIVED_SYMLINK;
} else {
return FileStatus.NOT_OURS;
}
} else {
return FileStatus.NON_EXISTENT;
}
}
}
@Override
public final File renameGrabbedDatafile(File file) throws QueueFileException {
String extension = FilenameUtils.getExtension(file.toString());
if (!extension.isEmpty()) {
extension = "." + extension;
}
for (int i = 0; i < RETRY; i++) {
File newFile = generateUniqueFile(extension, false);
if (!file.renameTo(newFile)) {
if (!Files.exists(newFile.toPath(), LinkOption.NOFOLLOW_LINKS)) {
throw new QueueFileException(
"Unable to rename file or symlink " + file + " to " + newFile);
}
} else {
return newFile;
}
}
throw new QueueFileException(RETRY + " attempts to rename file / symlink failed!");
}
}