ProdCompressionCacheSetup.java

package org.webpieces.router.impl.compression;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import javax.inject.Inject;

import org.webpieces.router.api.RouterConfig;
import org.webpieces.router.impl.StaticRoute;
import org.webpieces.router.impl.compression.MimeTypes.MimeTypeResult;
import org.webpieces.util.logging.Logger;
import org.webpieces.util.logging.LoggerFactory;
import org.webpieces.util.security.Security;

public class ProdCompressionCacheSetup implements CompressionCacheSetup {

	private static final Logger log = LoggerFactory.getLogger(ProdCompressionCacheSetup.class);
	
	private CompressionLookup lookup;
	private RouterConfig config;
	private MimeTypes mimeTypes;
	private List<String> encodings = new ArrayList<>();
	private FileUtil fileUtil;
	private Map<String, FileMeta> pathToFileMeta = new HashMap<>();

	@Inject
	public ProdCompressionCacheSetup(CompressionLookup lookup, RouterConfig config, MimeTypes mimeTypes, FileUtil fileUtil) {
		this.lookup = lookup;
		this.config = config;
		this.mimeTypes = mimeTypes;
		encodings.add(config.getStartupCompression());
		this.fileUtil = fileUtil;
	}
	
	public void setupCache(List<StaticRoute> staticRoutes) {
		if(config.getCachedCompressedDirectory() == null) {
			log.info("NOT setting up compressed cached directory so performance will not be as good");
			return;
		}
		
		log.info("setting up compressed cache directories");
		for(StaticRoute route : staticRoutes) {
			createCache(route);
		}
		log.info("all cached directories setup");
	}

	private void createCache(StaticRoute route) {
		File routeCache = route.getTargetCacheLocation();
		createDirectory(routeCache);
		
		File metaFile = new File(routeCache, "webpiecesMeta.properties");
		Properties p = load(metaFile);
		
		if(route.isFile()) {
			File file = new File(route.getFileSystemPath());
			log.info("setting up cache for file="+file);
			File destination = new File(routeCache, file.getName()+".gz");
			maybeAddFileToCache(p, file, destination, route.getFullPath());
		} else {
			File directory = new File(route.getFileSystemPath());
			log.info("setting up cache for directory="+directory);
			String urlPrefix = route.getFullPath();
			transferAndCompress(p, directory, routeCache, urlPrefix);
		}
		
		route.setHashMeta(p);
		store(metaFile, p);
	}

	private void store(File metaFile, Properties p) {
		try {
			FileOutputStream out = new FileOutputStream(metaFile);
			p.store(out, "file hashes for next time.  Single file format(key:urlPathOnly, value:hash), dir(key:urlPath+relativeFilePath, value:hash)");
		} catch(IOException e) {
			throw new RuntimeException(e);
		}
	}

	private Properties load(File metaFile) {
		try {
			Properties p = new Properties();
			if(!metaFile.exists())
				return p;
			p.load(new FileInputStream(metaFile));
			return p;
		} catch(IOException e) {
			throw new RuntimeException(e);
		}
	}

	private void transferAndCompress(Properties p, File directory, File destination, String urlPath) {
		File[] files = directory.listFiles();
		for(File f : files) {
			if(f.isDirectory()) {
				File newTarget = new File(destination, f.getName());
				createDirectory(newTarget);
				transferAndCompress(p, f, newTarget, urlPath+f.getName()+"/");
			} else {
				File newTarget = new File(destination, f.getName()+".gz");
				String path = urlPath+f.getName();
				maybeAddFileToCache(p, f, newTarget, path);
			}
		}
	}

	private void maybeAddFileToCache(Properties properties, File src, File destination, String urlPath) {
		String name = src.getName();
		int indexOf = name.lastIndexOf(".");
		if(indexOf < 0) {
			pathToFileMeta.put(urlPath, new FileMeta());
			return; //do nothing
		}
		String extension = name.substring(indexOf+1);
		
		MimeTypeResult mimeType = mimeTypes.extensionToContentType(extension, "application/octet-stream");
		Compression compression = lookup.createCompressionStream(encodings, extension, mimeType);
		if(compression == null) {
			pathToFileMeta.put(urlPath, new FileMeta());
			return;
		}

		//before we do the below, do a quick timestamp check to avoid reading in the files when not necessary
		long lastModifiedSrc = src.lastModified();
		long lastModified = destination.lastModified();
		//if hash is not there, the user may have changed the url so need to recalculate new hashes for new keys
		//There is a test for this...
		String previousHash = properties.getProperty(urlPath); 
		if(lastModified > lastModifiedSrc && previousHash != null) {
			log.info("timestamp later than src so skipping writing to="+destination);
			pathToFileMeta.put(urlPath, new FileMeta(previousHash));
			return; //no need to check anything as destination was written after this source file
		}
		
		try {
				byte[] allData = fileUtil.readFileContents(urlPath, src);
				String hash = Security.hash(allData);
				
				if(previousHash != null) {
					if(hash.equals(previousHash)) {
						if(!destination.exists())
							throw new IllegalStateException("Previously existing file is missing="+destination+" Your file cache was "
									+ "corrupted.  You will need to delete the whole cache directory");

						log.info("Previous file is the same, no need to compress to="+destination+" hash="+hash);
						pathToFileMeta.put(urlPath, new FileMeta(previousHash));
						return;
					}
				}

				//open, write, and close file with new data
				writeFile(destination, compression, allData, urlPath, src);
				//if file writing succeeded, set the hash
				properties.setProperty(urlPath, hash);
				
				FileMeta existing = pathToFileMeta.get(urlPath);
				if(existing != null)
					throw new IllegalStateException("this urlpath="+urlPath+" is referencing two files.  hash1="+existing.getHash()+" hash2="+hash
							+"  You should search your logs for this hash");
					
				pathToFileMeta.put(urlPath, new FileMeta(hash));
				
				log.info("compressed "+src.length()+" bytes to="+destination.length()+" to file="+destination+" hash="+hash);
				
		} catch (FileNotFoundException e) {
			throw new RuntimeException(e);
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	}

	private void writeFile(File destination, Compression compression, byte[] allData, String urlPath, File src) throws FileNotFoundException, IOException {
		FileOutputStream out = new FileOutputStream(destination);
		try(OutputStream compressionOut = compression.createCompressionStream(out)) 
		{
			fileUtil.writeFile(compressionOut, allData, urlPath, src);			
		}
	}

	private void createDirectory(File directoryToCreate) {
		if(directoryToCreate.exists()) {
			if(!directoryToCreate.isDirectory())
				throw new RuntimeException("File="+directoryToCreate+" is NOT a directory and we need a directory there...perhaps"
						+ " delete the cache and restart the server as something is corrupt");
			return;
		}
		
		boolean success = directoryToCreate.mkdirs();
		if(!success)
			throw new RuntimeException("Could not create cache directory="+directoryToCreate);
	}

	@Override
	public FileMeta relativeUrlToHash(String path) {
		return pathToFileMeta.get(path);
	}
}