/*
* Copyright 2011 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.gwt.dev.javac;
import com.google.gwt.core.ext.TreeLogger;
import com.google.gwt.core.ext.UnableToCompleteException;
import com.google.gwt.dev.jjs.impl.GwtAstBuilder;
import com.google.gwt.dev.util.log.speedtracer.DevModeEventType;
import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger;
import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event;
import com.google.gwt.util.tools.Utility;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* A class that manages a persistent cache of {@link CompilationUnit} instances.
* Writes out {@CompilationUnit} instances to a cache in a
* background thread.
* <p>
* The persistent cache is implemented as a directory of log files with a date
* timestamp. A new log file gets created each time a new PersistentUnitCache is
* instantiated, (once per invocation of the compiler or DevMode). The design is
* intended to support only a single PersistentUnitCache instance in the
* compiler at a time.
* <p>
* As new units are compiled, the cache data is appended to a log. This allows
* Java serialization to efficiently store references. The next time the cache
* is started, all logs are replayed and loaded into the cache in chronological
* order, with newer units taking precedence. A new cache file is created for
* any newly compiled units in this session. After a threshold of a certain
* number of files in the directory is reached
* {@link PersistentUnitCache#CACHE_FILE_THRESHOLD} , the cache files are
* consolidated back into a single file.
*
* <p>
* System Properties (see {@link UnitCacheFactory}).
*
* <ul>
* <li>gwt.persistentunitcache : enables the persistent cache (eventually will
* be default)</li>
* <li>gwt.persistentunitcachedir=<dir>: sets or overrides the cache directory</li>
* </ul>
*
* <p>
* Known Issues:
*
* <ul>
* <li>This design uses an eager cache to load every unit in the cache on the
* first reference to find() or add(). When the cache is large (10000 units), it
* uses lots of heap and takes 5-10 seconds. Once the PersistentUnitCache is
* created, it starts eagerly loading the cache in a background thread).</li>
*
* <li>Although units logged to disk with the same resource path are eventually
* cleaned up, the most recently compiled unit stays in the cache forever. This
* means that stale units that are no longer referenced will never be purged,
* unless by some external action (e.g. ant clean).</li>
*
* <li>Unless ant builds are made aware of the cache directory, the cache will
* persist if a user does an ant clean.</li>
* </ul>
*
*/
class PersistentUnitCache extends MemoryUnitCache {
/**
* A thread used when the cache is instantiated to load up cached units from
* the persistent store in the background. The
* {@link UnitCacheFactory#addUnit(CompilationUnit)} and
* {@link UnitCacheFactory#findUnit(String)} methods block if invoked before
* this thread finishes.
*/
private class UnitCacheMapLoader extends Thread {
private final CountDownLatch loadCompleteLatch = new CountDownLatch(1);
private final TreeLogger logger;
public UnitCacheMapLoader(TreeLogger logger) {
this.logger = logger;
setDaemon(true);
setName("UnitCacheLoader");
setPriority(Thread.NORM_PRIORITY);
}
public void await() {
try {
loadCompleteLatch.await();
} catch (InterruptedException ex) {
logger.log(TreeLogger.ERROR, "Interrupted waiting for PersistentUnitCache to load.", ex);
}
}
@Override
public void run() {
try {
loadUnitMap(logger);
} finally {
loadCompleteLatch.countDown();
if (logger.isLoggable(TreeLogger.TRACE)) {
logger
.log(TreeLogger.TRACE, "Loaded " + unitMap.size() + " units from persistent store.");
}
}
}
}
/**
* Used to pass messages to the unitWriteThread.
*/
private static class UnitWriteMessage {
private static final UnitWriteMessage DELETE_OLD_CACHE_FILES = new UnitWriteMessage();
private static final UnitWriteMessage SHUTDOWN_THREAD = new UnitWriteMessage();
private final UnitCacheEntry unitCacheEntry;
public UnitWriteMessage() {
unitCacheEntry = null;
}
public UnitWriteMessage(UnitCacheEntry unitCacheEntry) {
this.unitCacheEntry = unitCacheEntry;
}
}
/**
* Thread that reads units from a queue and writes out to a cache file for
* this session.
*/
private class UnitWriter extends Thread {
private final CountDownLatch shutDownLatch = new CountDownLatch(1);
private final TreeLogger logger;
private boolean errorLogged = false;
private Thread shutdownHook = new Thread() {
@Override
public void run() {
try {
doShutdown();
} catch (InterruptedException ex) {
// ignore
}
}
};
public UnitWriter(TreeLogger logger) {
this.logger = logger;
setDaemon(true);
setName("UnitWriteThread");
setPriority(Thread.MIN_PRIORITY);
Runtime.getRuntime().addShutdownHook(shutdownHook);
}
@Override
public void run() {
logger.log(TreeLogger.TRACE, "Starting UnitWriteThread.");
FileOutputStream fstream = null;
BufferedOutputStream bstream = null;
ObjectOutputStream stream = null;
try {
fstream = new FileOutputStream(currentCacheFile);
bstream = new BufferedOutputStream(fstream);
stream = new ObjectOutputStream(bstream);
} catch (IOException ex) {
logger.log(TreeLogger.ERROR, "Error creating cache " + currentCacheFile
+ ". Disabling cache.", ex);
}
int recentUnitsWritten = 0;
int totalUnitsWritten = 0;
try {
while (true) {
UnitWriteMessage msg = null;
try {
msg = unitWriteQueue.take();
} catch (InterruptedException e) {
// Allow shutdown to interrupt
break;
}
if (stream == null) {
// if there is no output stream, just ignore the unit
continue;
}
try {
if (msg != null) {
if (msg == UnitWriteMessage.DELETE_OLD_CACHE_FILES) {
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, "Wrote " + recentUnitsWritten
+ " units to persistent cache.");
}
recentUnitsWritten = 0;
deleteOldCacheFiles(logger, currentCacheFile);
} else if (msg == UnitWriteMessage.SHUTDOWN_THREAD) {
stream.flush();
assert unitWriteQueue.size() == 0;
break;
} else {
assert msg.unitCacheEntry.getOrigin() != UnitOrigin.ARCHIVE;
CompilationUnit unit = msg.unitCacheEntry.getUnit();
assert unit != null;
stream.writeObject(unit);
recentUnitsWritten++;
totalUnitsWritten++;
}
}
if (unitWriteQueue.isEmpty()) {
stream.flush();
}
} catch (IOException ex) {
if (!errorLogged) {
errorLogged = true;
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, "Error saving unit to file: "
+ currentCacheFile.getAbsolutePath(), ex);
}
}
}
}
} finally {
Utility.close(stream);
// Paranoia - close all streams
Utility.close(bstream);
Utility.close(fstream);
if (totalUnitsWritten == 0) {
// Remove useless empty output.
currentCacheFile.delete();
}
shutDownLatch.countDown();
logger.log(TreeLogger.TRACE, "Shutting down PersistentUnitCache thread");
}
}
/**
* Shutdown the thread and wait for it.
*/
private void doShutdown() throws InterruptedException {
// force the shutdown to finish after 5 seconds
unitWriteQueue.add(UnitWriteMessage.SHUTDOWN_THREAD);
// wait for shutdown
shutDownLatch.await(5000, TimeUnit.MILLISECONDS);
try {
Runtime.getRuntime().removeShutdownHook(shutdownHook);
} catch (IllegalStateException ex) {
// ignore.
}
}
}
/**
* Common prefix for creating directories and cache files.
*/
static final String UNIT_CACHE_PREFIX = "gwt-unitCache";
static final String CACHE_FILE_PREFIX = UNIT_CACHE_PREFIX + "-";
/**
* If there are more than this many files in the cache, clean up the old
* files.
*/
static final int CACHE_FILE_THRESHOLD = 10;
/**
* Used for communication to the unit write thread.
*/
private final BlockingQueue<UnitWriteMessage> unitWriteQueue =
new LinkedBlockingQueue<UnitWriteMessage>();
private final AtomicInteger addCount = new AtomicInteger(0);
private final UnitCacheMapLoader unitCacheMapLoader;
private final UnitWriter unitWriter;
private boolean cleanupHasRun = false;
/**
* A directory that ideally persists between invocations.
*/
private final File cacheDirectory;
/**
* Cache log file currently being written to.
*/
private File currentCacheFile;
PersistentUnitCache(TreeLogger logger, File cacheDir) throws UnableToCompleteException {
assert cacheDir != null;
this.cacheDirectory = new File(cacheDir, UNIT_CACHE_PREFIX);
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, "Persistent unit cache dir set to: "
+ this.cacheDirectory.getAbsolutePath());
}
if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) {
logger.log(TreeLogger.WARN, "Unable to initialize cache. Couldn't create directory "
+ cacheDirectory.getAbsolutePath() + ".");
throw new UnableToCompleteException();
}
long timestamp = System.currentTimeMillis();
do {
currentCacheFile =
new File(cacheDirectory, CACHE_FILE_PREFIX + String.format("%016X", timestamp++));
} while (currentCacheFile.exists());
// This isn't 100% reliable if multiple processes are in contention
try {
currentCacheFile.createNewFile();
} catch (IOException ex) {
logger.log(TreeLogger.WARN, "Unable to create new cache log file "
+ currentCacheFile.getAbsolutePath() + ".", ex);
throw new UnableToCompleteException();
}
unitCacheMapLoader = new UnitCacheMapLoader(logger);
unitCacheMapLoader.start();
unitWriter = new UnitWriter(logger);
unitWriter.start();
}
/**
* Enqueue a unit to be written by the background thread.
*/
@Override
public void add(CompilationUnit newUnit) {
unitCacheMapLoader.await();
super.add(newUnit);
UnitCacheEntry entry = unitMap.get(newUnit.getResourcePath());
addCount.getAndIncrement();
unitWriteQueue.add(new UnitWriteMessage(entry));
}
/**
* Cleans up old cache files in the directory, migrating everything previously
* loaded in them to the current cache file.
*
* Normally, only newly compiled units are written to the current log, but
* when it is time to cleanup, valid units from older log files need to be
* re-written.
*/
@Override
public void cleanup(TreeLogger logger) {
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, "Added " + addCount.intValue() + " units to persistent cache.");
}
addCount.set(0);
if (cleanupHasRun) {
return;
}
cleanupHasRun = true;
unitCacheMapLoader.await();
File[] cacheFiles = getCacheFiles();
if (cacheFiles.length < CACHE_FILE_THRESHOLD) {
return;
}
/*
* Resend all units read in from the in-memory cache to the writer thread.
* They will be re-written out and the old cache files removed.
*/
synchronized (unitMap) {
for (UnitCacheEntry unitCacheEntry : unitMap.values()) {
if (unitCacheEntry.getOrigin() == UnitOrigin.PERSISTENT) {
unitWriteQueue.add(new UnitWriteMessage(unitCacheEntry));
}
}
}
unitWriteQueue.add(UnitWriteMessage.DELETE_OLD_CACHE_FILES);
}
@Override
public CompilationUnit find(ContentId contentId) {
unitCacheMapLoader.await();
return super.find(contentId);
}
@Override
public CompilationUnit find(String resourcePath) {
unitCacheMapLoader.await();
return super.find(resourcePath);
}
/**
* Delete all cache files in the directory except for the currently open file.
*
* @param current Specifies the currently open cache file which will not be
* deleted.
*/
void deleteOldCacheFiles(TreeLogger logger, File current) {
assert current != null;
SpeedTracerLogger.Event deleteEvent = SpeedTracerLogger.start(DevModeEventType.DELETE_CACHE);
File[] filesToDelete = getCacheFiles();
if (filesToDelete == null) {
return;
}
if (logger.isLoggable(TreeLogger.INFO)) {
logger.log(TreeLogger.TRACE, "Purging cache files from " + cacheDirectory);
}
for (File toDelete : filesToDelete) {
if (!current.equals(toDelete)) {
toDelete.delete();
}
}
deleteEvent.end();
}
/**
* Finds all files matching a pattern in the cache directory.
*
* @return an array of sorted filenames. The file name pattern is such that
* sorting them alphabetically also sorts the files by age.
*/
File[] getCacheFiles() {
if (cacheDirectory.isDirectory()) {
File[] files = cacheDirectory.listFiles();
List<File> cacheFiles = new ArrayList<File>();
for (File file : files) {
if (file.getName().startsWith(CACHE_FILE_PREFIX)) {
cacheFiles.add(file);
}
}
File[] retFiles = cacheFiles.toArray(new File[cacheFiles.size()]);
Arrays.sort(retFiles);
return retFiles;
}
return new File[0];
}
/**
* For Unit testing - shutdown the persistent cache.
*/
void shutdown() throws InterruptedException {
unitWriter.doShutdown();
}
/**
* Load everything cached on disk into memory.
*/
private void loadUnitMap(TreeLogger logger) {
Event loadPersistentUnitEvent =
SpeedTracerLogger.start(DevModeEventType.LOAD_PERSISTENT_UNIT_CACHE);
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, "Looking for previously cached Compilation Units in "
+ cacheDirectory.getAbsolutePath());
}
try {
if (cacheDirectory.isDirectory() && cacheDirectory.canRead()) {
File[] files = getCacheFiles();
for (File cacheFile : files) {
FileInputStream fis = null;
BufferedInputStream bis = null;
ObjectInputStream inputStream = null;
if (cacheFile.equals(currentCacheFile)) {
continue;
}
boolean deleteCacheFile = false;
try {
fis = new FileInputStream(cacheFile);
bis = new BufferedInputStream(fis);
/*
* It is possible for the next call to throw an exception, leaving
* inputStream null and fis still live.
*/
inputStream = new ObjectInputStream(bis);
while (true) {
CachedCompilationUnit unit = (CachedCompilationUnit) inputStream.readObject();
if (unit == null) {
break;
}
if (unit.getTypesSerializedVersion() != GwtAstBuilder.getSerializationVersion()) {
continue;
}
UnitCacheEntry entry = new UnitCacheEntry(unit, UnitOrigin.PERSISTENT);
UnitCacheEntry existingEntry = unitMap.get(unit.getResourcePath());
/*
* Don't assume that an existing entry is stale - an entry might
* have been loaded already from another source like a
* CompilationUnitArchive that is more up to date. If the
* timestamps are the same, accept the latest version. If it turns
* out to be stale, it will be recompiled and the updated unit
* will win this test the next time the session starts.
*/
if (existingEntry != null && unit.getLastModified() >= existingEntry.getUnit().getLastModified()) {
super.remove(existingEntry.getUnit());
unitMap.put(unit.getResourcePath(), entry);
unitMapByContentId.put(unit.getContentId(), entry);
} else if (existingEntry == null) {
unitMap.put(unit.getResourcePath(), entry);
unitMapByContentId.put(unit.getContentId(), entry);
}
}
} catch (EOFException ex) {
// Go on to the next file.
} catch (IOException ex) {
deleteCacheFile = true;
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
+ cacheFile.getAbsolutePath() + " due to read error.", ex);
}
} catch (ClassNotFoundException ex) {
deleteCacheFile = true;
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
+ cacheFile.getAbsolutePath() + " due to deserialization error.", ex);
}
} finally {
Utility.close(inputStream);
Utility.close(bis);
Utility.close(fis);
}
if (deleteCacheFile) {
cacheFile.delete();
} else {
if (logger.isLoggable(TreeLogger.TRACE)) {
logger.log(TreeLogger.TRACE, cacheFile.getName() + ": Load complete");
}
}
}
} else {
logger
.log(TreeLogger.TRACE,
"Starting with empty Cache: CompilationUnit cache directory does not exist or is not readable.");
}
} finally {
loadPersistentUnitEvent.end();
}
}
}