package org.obo.app.util;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import org.apache.log4j.Logger;
/**
* A URLProxy can be used to maintain a local cache of URL resources. For HTTP
* URLs, it uses the "Last-Modified" and "ETag" headers to determine if the file
* needs to be re-downloaded.
*
* @author Jim Balhoff
*/
public class URLProxy {
private final File cacheFolder;
private static final String METADATA_FILENAME = ".proxy_metadata";
public enum CacheOption {
FORCE_CACHE, USE_CACHE, NO_CACHE;
}
/**
* Create a URLProxy which uses the given folder as its cache location.
*/
public URLProxy(File cacheLocation) {
if (!cacheLocation.exists()) {
cacheLocation.mkdirs();
}
if (!cacheLocation.isDirectory()) {
throw new IllegalArgumentException(
"Cache location must be a directory.");
}
this.cacheFolder = cacheLocation;
}
/**
* Returns true if the given URL is either not cached, or is cached and the
* cached data is out of date.
*/
public boolean isOutOfDate(URL url) throws IOException {
if (!this.isCached(url)) {
return true;
}
final URLConnection connection = url.openConnection();
final long lastModifiedRaw = connection.getLastModified();
final Date cacheDate = this.getCacheDate(connection.getURL());
if ((lastModifiedRaw > 0) && (cacheDate != null)) {
final Date lastModified = new Date(lastModifiedRaw);
return lastModified.after(cacheDate);
}
final String eTag = connection.getHeaderField("ETag");
final String cacheETag = this.getCacheETag(url);
if ((eTag != null) && (cacheETag != null)) {
return !eTag.equals(cacheETag);
}
return true;
}
/**
* Returns a local file containing the data at the given URL. A cached
* version will be used if it is not out of date. This calls get(URL url,
* CacheOption option) with CacheOption.USE_CACHE.
*/
public File get(URL url) throws IOException {
return this.get(url, CacheOption.USE_CACHE);
}
/**
* Returns a local file containing the data at the given URL. The data will
* be downloaded depending on the passed CacheOption. NO_CACHE will force a
* new version to be downloaded. USE_CACHE will download the file only if
* the cache is out of date. FORCE_CACHE will use the cached version even if
* it is out of date.
*/
public File get(URL url, CacheOption option) throws IOException {
if ((option.equals(CacheOption.NO_CACHE))
|| ((option.equals(CacheOption.USE_CACHE)) && (this
.isOutOfDate(url)))) {
log().info("Need to download from web: " + url);
this.downloadToCache(url);
}
return this.getCacheFile(url);
}
/**
* Returns true if a cached version of the given URL is available.
*/
public boolean isCached(URL url) {
return this.getCacheFile(url).exists();
}
/**
* Request the URLProxy to download and cache the given URL, regardless of
* its current cached status.
*/
public void downloadToCache(URL url) throws IOException {
URLConnection connection = url.openConnection();
if (connection instanceof HttpURLConnection) {
connection = followRedirects((HttpURLConnection)connection);
this.setCacheETag(url, connection.getHeaderField("ETag"));
this.setCacheDate(url, new Date(connection.getDate()));
} else {
this.setCacheDate(url, new Date(connection.getLastModified()));
}
final InputStream input = connection.getInputStream();
final ReadableByteChannel readChannel = Channels.newChannel(input);
final FileChannel writeChannel = (new FileOutputStream(this.getCacheFile(url))).getChannel();
final long BIG_NUM = 99999999999999l;
writeChannel.transferFrom(readChannel, 0, BIG_NUM);
writeChannel.close();
}
private HttpURLConnection followRedirects(HttpURLConnection connection) throws IOException {
final int status = connection.getResponseCode();
if (status == HttpURLConnection.HTTP_MOVED_TEMP
|| status == HttpURLConnection.HTTP_MOVED_PERM
|| status == HttpURLConnection.HTTP_SEE_OTHER) {
return (HttpURLConnection)new URL(connection.getHeaderField("Location")).openConnection();
} else return connection;
}
private File getCacheFile(URL url) {
return new File(this.cacheFolder, this.getCacheFileName(url));
}
private String getCacheFileName(URL url) {
return (String) this.getCacheMetadata(url).get("uuid");
}
private File getCacheMetadataFile() throws IOException {
final File file = new File(this.cacheFolder, METADATA_FILENAME);
if (!file.exists()) {
final FileWriter writer = new FileWriter(file);
writer.write("");
writer.close();
}
return file;
}
@SuppressWarnings("unchecked")
private Map<String, Object> getCacheMetadata() {
try {
final ObjectInputStream input = new ObjectInputStream(
new FileInputStream(this.getCacheMetadataFile()));
return (Map<String, Object>) (input.readObject());
} catch (EOFException e) {
// haven't serialized a map before
return new HashMap<String, Object>();
} catch (FileNotFoundException e) {
log().error("Unable to read download cache metadata", e);
} catch (IOException e) {
log().error("Unable to read download cache metadata", e);
} catch (ClassNotFoundException e) {
log().error("Unable to read download cache metadata", e);
}
return null;
}
@SuppressWarnings("unchecked")
private Map<String, Object> getCacheMetadata(URL url) {
final Map<String, Object> metadata = this.getCacheMetadata();
if (!metadata.containsKey(url.toString())) {
this.writeCacheMetadata(url, "uuid", UUID.randomUUID().toString());
}
return (Map<String, Object>) this.getCacheMetadata()
.get(url.toString());
}
/**
* All edits to the metadata dictionaries should be done through this
* method, which handles persistence. After editing the metadata, it should
* be refetched using getCacheMetadata.
*/
@SuppressWarnings("unchecked")
private void writeCacheMetadata(URL url, String key, Object value) {
try {
final Map<String, Object> metadata = this.getCacheMetadata();
final String urlKey = url.toString();
final Map<String, Object> urlMetadata;
if (metadata.containsKey(urlKey)) {
urlMetadata = (Map<String, Object>) (metadata.get(urlKey));
} else {
urlMetadata = new HashMap<String, Object>();
metadata.put(urlKey, urlMetadata);
}
urlMetadata.put(key, value);
final ObjectOutputStream output = new ObjectOutputStream(
new FileOutputStream(this.getCacheMetadataFile()));
output.writeObject(metadata);
output.close();
} catch (FileNotFoundException e) {
log().error("Unable to write download cache metadata", e);
} catch (IOException e) {
log().error("Unable to write download cache metadata", e);
}
}
private Date getCacheDate(URL url) {
return (Date) (this.getCacheMetadata(url).get("date"));
}
private void setCacheDate(URL url, Date date) {
this.writeCacheMetadata(url, "date", date);
}
private String getCacheETag(URL url) {
return (String) (this.getCacheMetadata(url).get("etag"));
}
private void setCacheETag(URL url, String eTag) {
this.writeCacheMetadata(url, "etag", eTag);
}
private Logger log() {
return Logger.getLogger(this.getClass());
}
}