/*
* Copyright (c) 2014 tabletoptool.com team.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the GNU Public License v3.0
* which accompanies this distribution, and is available at
* http://www.gnu.org/licenses/gpl.html
*
* Contributors:
* rptools.com team - initial implementation
* tabletoptool.com team - further development
*/
package com.t3.model;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import com.t3.MD5Key;
import com.t3.client.AppUtil;
import com.t3.client.TabletopTool;
import com.t3.persistence.FileUtil;
public class AssetLoader {
private static final Logger log = Logger.getLogger(AssetLoader.class);
public enum RepoState {
ACTIVE, BAD_URL, INDX_BAD_FORMAT, UNAVAILABLE
}
private static final File REPO_CACHE_DIR = AppUtil.getAppHome("repoindx");
private final ExecutorService retrievalThreadPool = Executors.newFixedThreadPool(3);
private final Set<MD5Key> requestedIdSet = new HashSet<MD5Key>();
private final Map<String, Map<String, String>> repositoryMap = new HashMap<String, Map<String, String>>();
private final Map<String, RepoState> repositoryStateMap = new HashMap<String, RepoState>();
public synchronized void addRepository(String repository) {
// Assume active, unless we find otherwise during setup
repositoryStateMap.put(repository, RepoState.ACTIVE);
repositoryMap.put(repository, getIndexMap(repository));
}
public synchronized void removeRepository(String repository) {
repositoryStateMap.remove(repository);
repositoryMap.remove(repository);
}
public synchronized void removeAllRepositories() {
repositoryMap.clear();
repositoryStateMap.clear();
}
public synchronized boolean isIdRequested(MD5Key id) {
return requestedIdSet.contains(id);
}
/**
* <p>
* This method returns the mapping from MD5Key to asset name on the server
* for the given repository.
* </p>
* <p>
* The return value is an immutable mapping so as to prevent any chance of
* the mapping being corrupted by the caller.
* </p>
*
* @param repo
* the name of the repository, probably from the campaign
* properties
* @return an immutable <code>Map<String, String></code>
*/
public Map<String, String> getRepositoryMap(String repo) {
return repositoryMap.get(repo);
}
/**
* <p>
* This method extracts an asset map from the given repository.
* </p>
* <p>
* It starts by checking to see if the repository index is already in the
* cache. If not, it makes a network connection and grabs it, calling
* {@link #storeIndexFile(String, byte[])} to store it into the cache.
* </p>
* <p>
* Once the index file has been located, {@link #parseIndex(List)} is called
* to convert the text file into a <code>Map<String, Sting></code> for
* the return value.
* </p>
*
* @param repository
* @return
*/
protected Map<String, String> getIndexMap(String repository) {
RepoState status = RepoState.ACTIVE;
Map<String, String> indexMap = new HashMap<String, String>();
try {
byte[] index = null;
if (!hasCurrentIndexFile(repository)) {
URL url = new URL(repository);
index = FileUtil.getBytes(url);
storeIndexFile(repository, index);
} else {
index = FileUtils.readFileToByteArray(getRepoIndexFile(repository));
}
indexMap = parseIndex(decode(index));
} catch (MalformedURLException e) {
log.error("Invalid repository URL: " + repository, e);
status = RepoState.BAD_URL;
} catch (IOException e) {
log.error("I/O error retrieving/saving index for '" + repository, e);
status = RepoState.UNAVAILABLE;
} catch (Throwable t) {
log.error("Could not retrieve index for '" + repository, t);
t.printStackTrace();
status = RepoState.UNAVAILABLE;
}
repositoryStateMap.put(repository, status);
return indexMap;
}
protected List<String> decode(byte[] indexData) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new ByteArrayInputStream(indexData))));
List<String> list = new ArrayList<String>();
String line = null;
while ((line = reader.readLine()) != null) {
list.add(line);
}
return list;
}
protected Map<String, String> parseIndex(List<String> index) {
Map<String, String> idxMap = new HashMap<String, String>();
for (String line : index) {
if (line == null) {
continue;
}
line = line.trim();
if (line.length() == 0) {
continue;
}
String id = line.substring(0, 32);
String ref = line.substring(33).trim();
idxMap.put(id, ref);
}
return idxMap;
}
/**
* <p>
* Converts the specified repository of assets into an index file that can
* be uploaded and used as the <b>index.gz</b> (after being compressed, of
* course).
* </p>
*
* @param repository
* @return
*/
protected byte[] createIndexFile(String repository) {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
Map<String, String> assets = repositoryMap.get(repository);
try(PrintWriter pw = new PrintWriter(bout)) {
for (String asset : assets.keySet())
pw.println(asset + " " + assets.get(asset));
}
return bout.toByteArray();
}
protected void storeIndexFile(String repository, byte[] data) throws IOException {
File file = getRepoIndexFile(repository);
FileUtils.writeByteArrayToFile(file, data);
}
protected boolean hasCurrentIndexFile(String repository) {
// TODO: make this check timestamps, or update once a day or something like that
return getRepoIndexFile(repository).exists();
}
protected File getRepoIndexFile(String repository) {
return new File(REPO_CACHE_DIR.getAbsolutePath() + "/" + new MD5Key(repository.getBytes()));
}
public synchronized void requestAsset(MD5Key id) {
retrievalThreadPool.submit(new ImageRetrievalRequest(id, createRequestQueue(id)));
requestedIdSet.add(id);
}
public synchronized void completeRequest(MD5Key id) {
requestedIdSet.remove(id);
}
protected List<String> createRequestQueue(MD5Key id) {
List<String> requestList = new LinkedList<String>();
for (java.util.Map.Entry<String, Map<String, String>> entry : repositoryMap.entrySet()) {
String repo = entry.getKey();
if (repositoryStateMap.get(repo) == RepoState.ACTIVE && entry.getValue().containsKey(id.toString())) {
requestList.add(repo);
}
}
return requestList;
}
private class ImageRetrievalRequest implements Runnable {
MD5Key id;
List<String> repositoryQueue;
public ImageRetrievalRequest(MD5Key id, List<String> repositoryQueue) {
this.id = id;
this.repositoryQueue = repositoryQueue;
}
@Override
public void run() {
while (repositoryQueue.size() > 0) {
String repo = repositoryQueue.remove(0);
Map<String, String> repoMap = repositoryMap.get(repo);
if (repoMap == null) {
// Must have been removed while we were asleep
continue;
}
String ref = repoMap.get(id.toString());
if (ref == null) {
// Must have updated while we were asleep
continue;
}
// Create the reference, need to work relative to the repo indx file
int split = repo.lastIndexOf('/');
try {
// make the URL http safe
String path = repo.substring(0, split + 1) + ref;
path = path.replaceAll(" ", "%20");
// Get the content
byte[] data = FileUtil.getBytes(new URL(path));
// Verify the content
MD5Key sum = new MD5Key(data);
if (!sum.equals(id)) {
// Bad file
// TODO: Does this mean it's time to update our cache of the index.gz?
// (See hasCurrentIndexFile() for the comment there.)
String msg = "Downloaded invalid file from: " + path;
log.warn(msg);
System.err.println(msg);
// Try a different repo
continue;
}
// Done
split = ref.lastIndexOf('/');
if (split >= 0) {
ref = ref.substring(split + 1);
}
// System.out.println("Got " + id + " from " + repo);
ref = FileUtil.getNameWithoutExtension(ref);
AssetManager.putAsset(new Asset(ref, data));
completeRequest(id);
return;
} catch (IOException ioe) {
// Well, try a different repo
// ioe.printStackTrace();
continue;
} catch (Throwable t) {
t.printStackTrace();
}
}
// Last resort, ask the MT server
// We can drop off the end of this runnable because it'll background load the
// image from the server
// System.out.println("Got " + id + " from MT");
TabletopTool.serverCommand().getAsset(id);
}
}
}