/* Copyright (c) 2013-2014 Boundless and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Distribution License v1.0 * which accompanies this distribution, and is available at * https://www.eclipse.org/org/documents/edl-v10.html * * Contributors: * Victor Olaya (Boundless) - initial implementation */ package org.locationtech.geogig.osm.internal.history; import static com.google.common.base.Preconditions.checkArgument; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.List; import java.util.concurrent.ExecutorService; import javax.xml.stream.XMLStreamException; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ContiguousSet; import com.google.common.collect.DiscreteDomain; import com.google.common.collect.Iterators; import com.google.common.collect.Range; import com.google.common.io.Closeables; /** * */ public class HistoryDownloader { private final long initialChangeset; private final long finalChangeset; private final ChangesetDownloader downloader; private Predicate<Changeset> filter = Predicates.alwaysTrue(); /** * @param osmAPIUrl api url, e.g. {@code http://api.openstreetmap.org/api/0.6}, * {@code file:/path/to/downloaded/changesets} * @param initialChangeset initial changeset id * @param finalChangeset final changeset id * @param preserveFiles */ public HistoryDownloader(final String osmAPIUrl, final File downloadFolder, long initialChangeset, long finalChangeset, ExecutorService executor) { checkArgument(initialChangeset > 0 && initialChangeset <= finalChangeset); this.initialChangeset = initialChangeset; this.finalChangeset = finalChangeset; this.downloader = new ChangesetDownloader(osmAPIUrl, downloadFolder, executor); } public void setChangesetFilter(Predicate<Changeset> filter) { this.filter = filter; } /** * */ private class ChangesSupplier implements Supplier<Optional<Iterator<Change>>> { private Supplier<Optional<File>> changesFile; /** * @param changesFile2 */ public ChangesSupplier(Supplier<Optional<File>> changesFile) { this.changesFile = changesFile; } @Override public Optional<Iterator<Change>> get() { return parseChanges(changesFile); } } /** * @return the next available changeset, or absent if reached the last one * @throws IOException * @throws InterruptedException */ public Iterator<Changeset> fetchChangesets() { Range<Long> range = Range.closed(initialChangeset, finalChangeset); ContiguousSet<Long> changesetIds = ContiguousSet.create(range, DiscreteDomain.longs()); final int fetchSize = 100; Iterator<List<Long>> partitions = Iterators.partition(changesetIds.iterator(), fetchSize); Function<List<Long>, Iterator<Changeset>> asChangesets = new Function<List<Long>, Iterator<Changeset>>() { @Override public Iterator<Changeset> apply(List<Long> batchIds) { Iterable<Changeset> changesets = downloader.fetchChangesets(batchIds); for (Changeset changeset : changesets) { if (filter.apply(changeset)) { Supplier<Optional<File>> changesFile; changesFile = downloader.fetchChanges(changeset.getId()); Supplier<Optional<Iterator<Change>>> changes = new ChangesSupplier( changesFile); changeset.setChanges(changes); } } return changesets.iterator(); } }; Iterator<Iterator<Changeset>> changesets = Iterators.transform(partitions, asChangesets); Iterator<Changeset> concat = Iterators.concat(changesets); return concat; } private Optional<Iterator<Change>> parseChanges(Supplier<Optional<File>> file) { final Optional<File> changesFile; try { changesFile = file.get(); } catch (RuntimeException e) { Throwable cause = e.getCause(); if (cause instanceof FileNotFoundException) { return Optional.absent(); } throw Throwables.propagate(e); } if (!changesFile.isPresent()) { return Optional.absent(); } final File actualFile = changesFile.get(); final InputStream stream = openStream(actualFile); final Iterator<Change> changes; ChangesetContentsScanner scanner = new ChangesetContentsScanner(); try { changes = scanner.parse(stream); } catch (XMLStreamException e) { throw Throwables.propagate(e); } Iterator<Change> iterator = new AbstractIterator<Change>() { @Override protected Change computeNext() { if (!changes.hasNext()) { Closeables.closeQuietly(stream); actualFile.delete(); actualFile.getParentFile().delete(); return super.endOfData(); } return changes.next(); } }; return Optional.of(iterator); } private InputStream openStream(File file) { InputStream stream; try { stream = new BufferedInputStream(new FileInputStream(file), 4096); } catch (FileNotFoundException e) { throw Throwables.propagate(e); } return stream; } }