// This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.replication.v0_6;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.OsmosisConstants;
import org.openstreetmap.osmosis.core.merge.common.ConflictResolutionMethod;
import org.openstreetmap.osmosis.core.pipeline.common.TaskRunner;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSink;
import org.openstreetmap.osmosis.core.task.v0_6.RunnableChangeSource;
import org.openstreetmap.osmosis.core.time.DateParser;
import org.openstreetmap.osmosis.core.util.FileBasedLock;
import org.openstreetmap.osmosis.replication.common.TimestampTracker;
import org.openstreetmap.osmosis.replication.v0_6.impl.ChangesetFileNameFormatter;
import org.openstreetmap.osmosis.replication.v0_6.impl.IntervalDownloaderConfiguration;
import org.openstreetmap.osmosis.set.v0_6.ChangeMerger;
import org.openstreetmap.osmosis.xml.common.CompressionMethod;
import org.openstreetmap.osmosis.xml.v0_6.XmlChangeReader;
/**
* Downloads a set of change files from a HTTP server, and merges them into a
* single output stream. It tracks the intervals covered by the current files
* and stores the current timestamp between invocations forming the basis of a
* replication mechanism.
*
* @author Brett Henderson
*/
public class IntervalDownloader implements RunnableChangeSource {
private static final Logger LOG = Logger.getLogger(IntervalDownloader.class.getName());
private static final String LOCK_FILE = "download.lock";
private static final String CONFIG_FILE = "configuration.txt";
private static final String TSTAMP_FILE = "timestamp.txt";
private static final String TSTAMP_NEW_FILE = "timestamp-new.txt";
private static final String SERVER_TSTAMP_FILE = "timestamp.txt";
private ChangeSink changeSink;
private String taskId;
private File workingDirectory;
private DateParser dateParser;
/**
* Creates a new instance.
*
* @param taskId
* The identifier for the task, this is required because the
* names of threads created by this task will use this name as a
* prefix.
* @param workingDirectory
* The directory containing configuration and tracking files.
*/
public IntervalDownloader(String taskId, File workingDirectory) {
this.taskId = taskId;
this.workingDirectory = workingDirectory;
dateParser = new DateParser();
}
/**
* {@inheritDoc}
*/
@Override
public void setChangeSink(ChangeSink changeSink) {
this.changeSink = changeSink;
}
/**
* Retrieves the latest timestamp from the server.
*
* @param baseUrl
* The url of the directory containing change files.
* @return The timestamp.
*/
private Date getServerTimestamp(URL baseUrl) {
URL timestampUrl;
try {
timestampUrl = new URL(baseUrl, SERVER_TSTAMP_FILE);
} catch (MalformedURLException e) {
throw new OsmosisRuntimeException("The server timestamp URL could not be created.", e);
}
try {
Date result;
URLConnection connection = timestampUrl.openConnection();
connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes
connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes
connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION);
try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
result = dateParser.parse(reader.readLine());
}
return result;
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to read the timestamp from the server.", e);
}
}
/**
* Downloads the file from the server with the specified name and writes it
* to a local temporary file.
*
* @param fileName
* The name of the file to download.
* @param baseUrl
* The url of the directory containing change files.
* @return The temporary file containing the downloaded data.
*/
private File downloadChangesetFile(String fileName, URL baseUrl) {
URL changesetUrl;
try {
changesetUrl = new URL(baseUrl, fileName);
} catch (MalformedURLException e) {
throw new OsmosisRuntimeException("The server file URL could not be created.", e);
}
try {
File outputFile;
// Open an input stream for the changeset file on the server.
URLConnection connection = changesetUrl.openConnection();
connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes
connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes
try (BufferedInputStream source = new BufferedInputStream(connection.getInputStream(), 65536)) {
// Create a temporary file to write the data to.
outputFile = File.createTempFile("change", null);
// Open a output stream for the destination file.
try (BufferedOutputStream sink = new BufferedOutputStream(new FileOutputStream(outputFile), 65536)) {
// Download the file.
byte[] buffer = new byte[65536];
for (int bytesRead = source.read(buffer); bytesRead > 0; bytesRead = source.read(buffer)) {
sink.write(buffer, 0, bytesRead);
}
}
}
return outputFile;
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to read the changeset file " + fileName + " from the server.", e);
}
}
/**
* Downloads the changeset files from the server and writes their contents
* to the output task.
*/
private void download() {
IntervalDownloaderConfiguration configuration;
TimestampTracker timestampTracker;
ChangesetFileNameFormatter fileNameFormatter;
Date currentTime;
Date maximumTime;
URL baseUrl;
int maxDownloadCount;
int downloadCount;
ArrayList<File> tmpFileList;
ArrayList<RunnableChangeSource> tasks;
ArrayList<TaskRunner> taskRunners;
boolean tasksSuccessful;
// Instantiate utility objects.
configuration = new IntervalDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
timestampTracker = new TimestampTracker(
new File(workingDirectory, TSTAMP_FILE),
new File(workingDirectory, TSTAMP_NEW_FILE)
);
fileNameFormatter = new ChangesetFileNameFormatter(
configuration.getChangeFileBeginFormat(),
configuration.getChangeFileEndFormat()
);
// Create the base url.
try {
baseUrl = new URL(configuration.getBaseUrl());
} catch (MalformedURLException e) {
throw new OsmosisRuntimeException(
"Unable to convert URL string (" + configuration.getBaseUrl() + ") into a URL.", e);
}
tmpFileList = new ArrayList<File>();
// Load the current time from the timestamp tracking file.
currentTime = timestampTracker.getTime();
// Load the latest timestamp from the server.
maximumTime = getServerTimestamp(baseUrl);
// Process until all files have been retrieved from the server.
maxDownloadCount = configuration.getMaxDownloadCount();
downloadCount = 0;
while ((maxDownloadCount == 0 || downloadCount < maxDownloadCount) && currentTime.before(maximumTime)) {
Date nextTime;
String downloadFileName;
// Calculate the end of the next time interval.
nextTime = new Date(currentTime.getTime() + configuration.getIntervalLength());
// Generate the filename to be retrieved from the server.
downloadFileName = fileNameFormatter.generateFileName(currentTime, nextTime);
// Download the changeset from the server.
tmpFileList.add(downloadChangesetFile(downloadFileName, baseUrl));
// Move the current time to the next interval.
currentTime = nextTime;
// Increment the current download count.
downloadCount++;
}
// Generate a set of tasks for loading the change files and merge them
// into a single change stream.
tasks = new ArrayList<RunnableChangeSource>();
for (File tmpFile : tmpFileList) {
XmlChangeReader changeReader;
// Generate a change reader task for the current task.
changeReader = new XmlChangeReader(
tmpFile,
true,
CompressionMethod.GZip
);
// If tasks already exist, a change merge task must be used to merge
// existing output with this task output, otherwise this task can be
// added to the list directly.
if (tasks.size() > 0) {
ChangeMerger changeMerger;
// Create a new change merger merging the last task output with the current task.
changeMerger = new ChangeMerger(ConflictResolutionMethod.LatestSource, 10);
// Connect the inputs of this merger to the most recent change
// output and the new change output.
tasks.get(tasks.size() - 1).setChangeSink(changeMerger.getChangeSink(0));
changeReader.setChangeSink(changeMerger.getChangeSink(1));
tasks.add(changeReader);
tasks.add(changeMerger);
} else {
tasks.add(changeReader);
}
}
// We only need to execute sub-threads if tasks exist, otherwise we must
// notify the sink that we have completed.
if (tasks.size() > 0) {
// Connect the last task to the change sink.
tasks.get(tasks.size() - 1).setChangeSink(changeSink);
// Create task runners for each of the tasks to provide thread
// management.
taskRunners = new ArrayList<TaskRunner>(tasks.size());
for (int i = 0; i < tasks.size(); i++) {
taskRunners.add(new TaskRunner(tasks.get(i), "Thread-" + taskId + "-worker" + i));
}
// Launch all of the tasks.
for (int i = 0; i < taskRunners.size(); i++) {
TaskRunner taskRunner;
taskRunner = taskRunners.get(i);
LOG.fine("Launching changeset worker + " + i + " in a new thread.");
taskRunner.start();
}
// Wait for all the tasks to complete.
tasksSuccessful = true;
for (int i = 0; i < taskRunners.size(); i++) {
TaskRunner taskRunner;
taskRunner = taskRunners.get(i);
LOG.fine("Waiting for changeset worker " + i + " to complete.");
try {
taskRunner.join();
} catch (InterruptedException e) {
// We are already in an error condition so log and continue.
LOG.log(Level.WARNING, "The wait for task completion was interrupted.", e);
}
if (!taskRunner.isSuccessful()) {
LOG.log(Level.SEVERE, "Changeset worker " + i + " failed", taskRunner.getException());
tasksSuccessful = false;
}
}
} else {
changeSink.complete();
tasksSuccessful = true;
}
// Remove the temporary files.
for (File tmpFile : tmpFileList) {
if (!tmpFile.delete()) {
LOG.warning("Unable to delete file " + tmpFile.getName());
}
}
if (!tasksSuccessful) {
throw new OsmosisRuntimeException("One or more changeset workers failed.");
}
// Update the timestamp tracker.
timestampTracker.setTime(currentTime);
}
/**
* {@inheritDoc}
*/
@Override
public void run() {
FileBasedLock fileLock;
fileLock = new FileBasedLock(new File(workingDirectory, LOCK_FILE));
try {
changeSink.initialize(Collections.<String, Object>emptyMap());
fileLock.lock();
download();
fileLock.unlock();
} finally {
changeSink.close();
fileLock.close();
}
}
}