/* This code is part of Freenet. It is distributed under the GNU General
* Public License, version 2 (or at your option any later version). See
* http://www.gnu.org/ for further details of the GPL. */
package freenet.support.io;
import static java.util.concurrent.TimeUnit.MINUTES;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.ref.WeakReference;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.Queue;
import java.util.Random;
import java.util.concurrent.LinkedBlockingQueue;
import freenet.client.async.ClientContext;
import freenet.crypt.EncryptedRandomAccessBucket;
import freenet.crypt.EncryptedRandomAccessBuffer;
import freenet.crypt.EncryptedRandomAccessBufferType;
import freenet.crypt.MasterSecret;
import freenet.support.Executor;
import freenet.support.LogThresholdCallback;
import freenet.support.Logger;
import freenet.support.Logger.LogLevel;
import freenet.support.SizeUtil;
import freenet.support.TimeUtil;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
import freenet.support.api.LockableRandomAccessBuffer;
import freenet.support.api.LockableRandomAccessBufferFactory;
import freenet.support.api.RandomAccessBucket;
/**
* Temporary Bucket Factory
*
* Buckets created by this factory can be either:
* - ArrayBuckets
* OR
* - FileBuckets
*
* ArrayBuckets are used if and only if:
* 1) there is enough room remaining on the pool (@see maxRamUsed and @see bytesInUse)
* 2) the initial size is smaller than (@maxRAMBucketSize)
*
* Depending on how they are used they might switch from one type to another transparently.
*
* Currently they are two factors considered for a migration:
* - if they are long-lived or not (@see RAMBUCKET_MAX_AGE)
* - if their size is over RAMBUCKET_CONVERSION_FACTOR*maxRAMBucketSize
*/
public class TempBucketFactory implements BucketFactory, LockableRandomAccessBufferFactory {
public final static long defaultIncrement = 4096;
public final static float DEFAULT_FACTOR = 1.25F;
private final FilenameGenerator filenameGenerator;
private final PooledFileRandomAccessBufferFactory underlyingDiskRAFFactory;
private final DiskSpaceCheckingRandomAccessBufferFactory diskRAFFactory;
private volatile long minDiskSpace;
private long bytesInUse = 0;
private final Executor executor;
private volatile boolean reallyEncrypt;
private final MasterSecret secret;
/** How big can the defaultSize be for us to consider using RAMBuckets? */
private long maxRAMBucketSize;
/** How much memory do we dedicate to the RAMBucketPool? (in bytes) */
private long maxRamUsed;
/** How old is a long-lived RAMBucket? */
private final static long RAMBUCKET_MAX_AGE = MINUTES.toMillis(5);
/** How many times the maxRAMBucketSize can a RAMBucket be before it gets migrated? */
final static int RAMBUCKET_CONVERSION_FACTOR = 4;
final static boolean TRACE_BUCKET_LEAKS = false;
private static volatile boolean logMINOR;
static {
Logger.registerLogThresholdCallback(new LogThresholdCallback(){
@Override
public void shouldUpdate(){
logMINOR = Logger.shouldLog(LogLevel.MINOR, this);
}
});
}
private interface Migratable {
long creationTime();
boolean migrateToDisk() throws IOException;
};
public class TempBucket implements Bucket, Migratable, RandomAccessBucket {
/** The underlying bucket itself */
private RandomAccessBucket currentBucket;
/** We have to account the size of the underlying bucket ourself in order to be able to access it fast */
private long currentSize;
/** Has an OutputStream been opened at some point? */
private boolean hasWritten;
/** A link to the "real" underlying outputStream, even if we migrated */
private OutputStream os = null;
/** All the open-streams to reset or close on migration or free() */
private final ArrayList<TempBucketInputStream> tbis;
/** An identifier used to know when to deprecate the InputStreams */
private short osIndex;
/** A timestamp used to evaluate the age of the bucket and maybe consider it for a migration */
public final long creationTime;
private boolean hasBeenFreed = false;
private final Throwable tracer;
public TempBucket(long now, RandomAccessBucket cur) {
if(cur == null)
throw new NullPointerException();
if (TRACE_BUCKET_LEAKS)
tracer = new Throwable();
else
tracer = null;
this.currentBucket = cur;
this.creationTime = now;
this.osIndex = 0;
this.tbis = new ArrayList<TempBucketInputStream>(1);
if(logMINOR) Logger.minor(TempBucket.class, "Created "+this, new Exception("debug"));
}
private synchronized void closeInputStreams(boolean forFree) {
for(ListIterator<TempBucketInputStream> i = tbis.listIterator(); i.hasNext();) {
TempBucketInputStream is = i.next();
if(forFree) {
i.remove();
try {
is.close();
} catch (IOException e) {
Logger.error(this, "Caught "+e+" closing "+is);
}
} else {
try {
is._maybeResetInputStream();
} catch(IOException e) {
i.remove();
Closer.close(is);
}
}
}
}
/** A blocking method to force-migrate from a RAMBucket to a FileBucket */
public final boolean migrateToDisk() throws IOException {
Bucket toMigrate = null;
long size;
synchronized(this) {
if(!isRAMBucket() || hasBeenFreed)
// Nothing to migrate! We don't want to switch back to ram, do we?
return false;
toMigrate = currentBucket;
RandomAccessBucket tempFB = _makeFileBucket();
size = currentSize;
if(os != null) {
os.flush();
os.close();
// DO NOT INCREMENT THE osIndex HERE!
os = tempFB.getOutputStreamUnbuffered();
if(size > 0)
BucketTools.copyTo(toMigrate, os, size);
} else {
if(size > 0) {
OutputStream temp = tempFB.getOutputStreamUnbuffered();
try {
BucketTools.copyTo(toMigrate, temp, size);
} finally {
temp.close();
}
}
}
if(toMigrate.isReadOnly())
tempFB.setReadOnly();
closeInputStreams(false);
currentBucket = tempFB;
// We need streams to be reset to point to the new bucket
}
if(logMINOR)
Logger.minor(this, "We have migrated "+toMigrate.hashCode());
synchronized(ramBucketQueue) {
ramBucketQueue.remove(getReference());
}
// We can free it on-thread as it's a rambucket
toMigrate.free();
// Might have changed already so we can't rely on currentSize!
_hasFreed(size);
return true;
}
public synchronized final boolean isRAMBucket() {
return (currentBucket instanceof ArrayBucket);
}
@Override
public OutputStream getOutputStream() throws IOException {
return new BufferedOutputStream(getOutputStreamUnbuffered());
}
@Override
public synchronized OutputStream getOutputStreamUnbuffered() throws IOException {
if(os != null)
throw new IOException("Only one OutputStream per bucket on "+this+" !");
if(hasBeenFreed) throw new IOException("Already freed");
// Hence we don't need to reset currentSize / _hasTaken() if a bucket is reused.
// FIXME we should migrate to disk rather than throwing.
hasWritten = true;
OutputStream tos = new TempBucketOutputStream(++osIndex);
if(logMINOR)
Logger.minor(this, "Got "+tos+" for "+this, new Exception());
return tos;
}
private class TempBucketOutputStream extends OutputStream {
long lastCheckedSize = 0;
long CHECK_DISK_EVERY = 4096;
boolean closed = false;
TempBucketOutputStream(short idx) throws IOException {
if(os == null)
os = currentBucket.getOutputStreamUnbuffered();
}
private void _maybeMigrateRamBucket(long futureSize) throws IOException {
if (closed) {
return;
}
if(isRAMBucket()) {
boolean shouldMigrate = false;
boolean isOversized = false;
if(futureSize >= Math.min(Integer.MAX_VALUE, maxRAMBucketSize * RAMBUCKET_CONVERSION_FACTOR)) {
isOversized = true;
shouldMigrate = true;
} else if ((futureSize - currentSize) + bytesInUse >= maxRamUsed)
shouldMigrate = true;
if(shouldMigrate) {
if(logMINOR) {
if(isOversized)
Logger.minor(this, "The bucket "+TempBucket.this+" is over "+SizeUtil.formatSize(maxRAMBucketSize*RAMBUCKET_CONVERSION_FACTOR)+": we will force-migrate it to disk.");
else
Logger.minor(this, "The bucketpool is full: force-migrate before we go over the limit");
}
migrateToDisk();
}
} else {
// Check for excess disk usage.
if(futureSize - lastCheckedSize >= CHECK_DISK_EVERY) {
if(filenameGenerator.getDir().getUsableSpace() + (futureSize - currentSize) <
minDiskSpace)
throw new InsufficientDiskSpaceException();
lastCheckedSize = futureSize;
}
}
}
@Override
public final void write(int b) throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) throw new IOException("Already freed");
long futureSize = currentSize + 1;
_maybeMigrateRamBucket(futureSize);
os.write(b);
currentSize = futureSize;
if(isRAMBucket()) // We need to re-check because it might have changed!
_hasTaken(1);
}
}
@Override
public final void write(byte b[], int off, int len) throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) throw new IOException("Already freed");
long futureSize = currentSize + len;
_maybeMigrateRamBucket(futureSize);
os.write(b, off, len);
currentSize = futureSize;
if(isRAMBucket()) // We need to re-check because it might have changed!
_hasTaken(len);
}
}
@Override
public final void flush() throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) return;
_maybeMigrateRamBucket(currentSize);
if(!closed)
os.flush();
}
}
@Override
public final void close() throws IOException {
synchronized(TempBucket.this) {
if(closed) return;
_maybeMigrateRamBucket(currentSize);
os.flush();
os.close();
os = null;
closed = true;
}
}
}
@Override
public InputStream getInputStream() throws IOException {
return new BufferedInputStream(getInputStreamUnbuffered());
}
@Override
public synchronized InputStream getInputStreamUnbuffered() throws IOException {
if(!hasWritten)
throw new IOException("No OutputStream has been openned! Why would you want an InputStream then?");
if(hasBeenFreed) throw new IOException("Already freed");
TempBucketInputStream is = new TempBucketInputStream(osIndex);
tbis.add(is);
if(logMINOR)
Logger.minor(this, "Got "+is+" for "+this, new Exception());
return is;
}
private class TempBucketInputStream extends InputStream {
/** The current InputStream we use from the underlying bucket */
private InputStream currentIS;
/** Keep a counter to know where we are on the stream (useful when we have to reset and skip) */
private long index = 0;
/** Will change if a new OutputStream is openned: used to detect deprecation */
private final short idx;
TempBucketInputStream(short idx) throws IOException {
this.idx = idx;
this.currentIS = currentBucket.getInputStreamUnbuffered();
}
public void _maybeResetInputStream() throws IOException {
if(idx != osIndex)
close();
else {
Closer.close(currentIS);
currentIS = currentBucket.getInputStreamUnbuffered();
long toSkip = index;
while(toSkip > 0) {
toSkip -= currentIS.skip(toSkip);
}
}
}
@Override
public final int read() throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) throw new IOException("Already freed");
int toReturn = currentIS.read();
if(toReturn != -1)
index++;
return toReturn;
}
}
@Override
public int read(byte b[]) throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) throw new IOException("Already freed");
return read(b, 0, b.length);
}
}
@Override
public int read(byte b[], int off, int len) throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) throw new IOException("Already freed");
int toReturn = currentIS.read(b, off, len);
if(toReturn > 0)
index += toReturn;
return toReturn;
}
}
@Override
public long skip(long n) throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) throw new IOException("Already freed");
long skipped = currentIS.skip(n);
index += skipped;
return skipped;
}
}
@Override
public int available() throws IOException {
synchronized(TempBucket.this) {
if(hasBeenFreed) throw new IOException("Already freed");
return currentIS.available();
}
}
@Override
public boolean markSupported() {
return false;
}
@Override
public final void close() throws IOException {
synchronized(TempBucket.this) {
Closer.close(currentIS);
tbis.remove(this);
}
}
}
@Override
public synchronized String getName() {
return currentBucket.getName();
}
@Override
public synchronized long size() {
return currentSize;
}
@Override
public synchronized boolean isReadOnly() {
return currentBucket.isReadOnly();
}
@Override
public synchronized void setReadOnly() {
currentBucket.setReadOnly();
}
@Override
public synchronized void free() {
Bucket cur;
synchronized(this) {
if(hasBeenFreed) return;
hasBeenFreed = true;
Closer.close(os);
closeInputStreams(true);
if(isRAMBucket()) {
// If it's in memory we must free before removing from the queue.
currentBucket.free();
_hasFreed(currentSize);
synchronized(ramBucketQueue) {
ramBucketQueue.remove(getReference());
}
return;
} else {
// Better to free outside the lock if it's not in-memory.
cur = currentBucket;
}
}
cur.free();
}
/** Called only by TempRandomAccessBuffer */
private synchronized void onFreed() {
hasBeenFreed = true;
}
@Override
public RandomAccessBucket createShadow() {
return currentBucket.createShadow();
}
private WeakReference<Migratable> weakRef = new WeakReference<Migratable>(this);
public WeakReference<Migratable> getReference() {
return weakRef;
}
@Override
protected void finalize() throws Throwable {
// If it's been converted to a TempRandomAccessBuffer, finalize() will only be called
// if *neither* object is reachable.
if (!hasBeenFreed) {
if (TRACE_BUCKET_LEAKS)
Logger.error(this, "TempBucket not freed, size=" + size() + ", isRAMBucket=" + isRAMBucket()+" : "+this, tracer);
else
Logger.error(this, "TempBucket not freed, size=" + size() + ", isRAMBucket=" + isRAMBucket()+" : "+this);
free();
}
super.finalize();
}
@Override
public long creationTime() {
return creationTime;
}
@Override
public void onResume(ClientContext context) {
// Not persistent.
throw new IllegalStateException();
}
@Override
public void storeTo(DataOutputStream dos) throws IOException {
throw new UnsupportedOperationException(); // Not persistent.
}
@Override
public LockableRandomAccessBuffer toRandomAccessBuffer() throws IOException {
synchronized(this) {
if(hasBeenFreed) throw new IOException("Already freed");
if(os != null) throw new IOException("Can't migrate with open OutputStream's");
if(!tbis.isEmpty()) throw new IOException("Can't migrate with open InputStream's");
setReadOnly();
TempRandomAccessBuffer raf = new TempRandomAccessBuffer(currentBucket.toRandomAccessBuffer(), creationTime, !isRAMBucket(), this);
if(isRAMBucket()) {
synchronized(ramBucketQueue) {
// No change in space usage.
ramBucketQueue.remove(getReference());
ramBucketQueue.add(raf.getReference());
}
}
currentBucket = new RAFBucket(raf);
return raf;
}
}
/** Only for testing */
synchronized Bucket getUnderlying() {
return currentBucket;
}
}
// Storage accounting disabled by default.
public TempBucketFactory(Executor executor, FilenameGenerator filenameGenerator, long maxBucketSizeKeptInRam, long maxRamUsed, Random weakPRNG, boolean reallyEncrypt, long minDiskSpace, MasterSecret masterSecret) {
this.filenameGenerator = filenameGenerator;
this.maxRamUsed = maxRamUsed;
this.maxRAMBucketSize = maxBucketSizeKeptInRam;
this.reallyEncrypt = reallyEncrypt;
this.executor = executor;
this.underlyingDiskRAFFactory = new PooledFileRandomAccessBufferFactory(filenameGenerator, weakPRNG);
underlyingDiskRAFFactory.enableCrypto(reallyEncrypt);
this.minDiskSpace = minDiskSpace;
this.diskRAFFactory = new DiskSpaceCheckingRandomAccessBufferFactory(underlyingDiskRAFFactory,
filenameGenerator.getDir(), minDiskSpace - maxRamUsed);
this.secret = masterSecret;
}
@Override
public RandomAccessBucket makeBucket(long size) throws IOException {
return makeBucket(size, DEFAULT_FACTOR, defaultIncrement);
}
public RandomAccessBucket makeBucket(long size, float factor) throws IOException {
return makeBucket(size, factor, defaultIncrement);
}
private synchronized void _hasTaken(long size) {
bytesInUse += size;
}
private synchronized void _hasFreed(long size) {
bytesInUse -= size;
}
public synchronized long getRamUsed() {
return bytesInUse;
}
public synchronized void setMaxRamUsed(long size) {
maxRamUsed = size;
}
public synchronized long getMaxRamUsed() {
return maxRamUsed;
}
public synchronized void setMaxRAMBucketSize(long size) {
maxRAMBucketSize = size;
diskRAFFactory.setMinDiskSpace(minDiskSpace - maxRamUsed);
}
public synchronized long getMaxRAMBucketSize() {
return maxRAMBucketSize;
}
public void setEncryption(boolean value) {
reallyEncrypt = value;
underlyingDiskRAFFactory.enableCrypto(value);
}
public synchronized void setMinDiskSpace(long min) {
minDiskSpace = min;
diskRAFFactory.setMinDiskSpace(minDiskSpace - maxRamUsed);
}
public boolean isEncrypting() {
return reallyEncrypt;
}
static final double MAX_USAGE_LOW = 0.8;
static final double MAX_USAGE_HIGH = 0.9;
public static final EncryptedRandomAccessBufferType CRYPT_TYPE = EncryptedRandomAccessBufferType.ChaCha128;
/**
* Create a temp bucket
*
* @param size
* Maximum size
* @param factor
* Factor to increase size by when need more space
* @return A temporary Bucket
* @exception IOException
* If it is not possible to create a temp bucket due to an
* I/O error
*/
public TempBucket makeBucket(long size, float factor, long increment) throws IOException {
RandomAccessBucket realBucket = null;
boolean useRAMBucket = false;
long now = System.currentTimeMillis();
synchronized(this) {
if((size > 0) && (size <= maxRAMBucketSize) && (bytesInUse < maxRamUsed) && (bytesInUse + size <= maxRamUsed)) {
useRAMBucket = true;
}
if(bytesInUse >= maxRamUsed * MAX_USAGE_HIGH && !runningCleaner) {
runningCleaner = true;
executor.execute(cleaner);
}
}
// Do we want a RAMBucket or a FileBucket?
realBucket = (useRAMBucket ? new ArrayBucket() : _makeFileBucket());
TempBucket toReturn = new TempBucket(now, realBucket);
if(useRAMBucket) { // No need to consider them for migration if they can't be migrated
synchronized(ramBucketQueue) {
ramBucketQueue.add(toReturn.getReference());
}
} else {
// If we know the disk space requirement in advance, check it.
if(size != -1 && size != Long.MAX_VALUE) {
if(filenameGenerator.getDir().getUsableSpace() + size < minDiskSpace)
throw new InsufficientDiskSpaceException();
}
}
return toReturn;
}
boolean runningCleaner = false;
private final Runnable cleaner = new Runnable() {
@Override
public void run() {
boolean saidSo = false;
try {
long now = System.currentTimeMillis();
// First migrate all the old buckets.
while(true) {
try {
cleanBucketQueue(now, false);
} catch (InsufficientDiskSpaceException e) {
if(!saidSo) {
Logger.error(this, "Insufficient disk space to migrate in-RAM buckets to disk!");
System.err.println("Out of disk space!");
saidSo = true;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e1) {
// Ignore.
}
continue;
}
break;
}
saidSo = false;
while(true) {
// Now migrate buckets until usage is below the lower threshold.
synchronized(TempBucketFactory.this) {
if(bytesInUse <= maxRamUsed * MAX_USAGE_LOW) return;
}
try {
if(!cleanBucketQueue(System.currentTimeMillis(), true)) return;
} catch (InsufficientDiskSpaceException e) {
if(!saidSo) {
Logger.error(this, "Insufficient disk space to migrate in-RAM buckets to disk!");
System.err.println("Out of disk space!");
saidSo = true;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e1) {
// Ignore.
}
}
}
} finally {
synchronized(TempBucketFactory.this) {
runningCleaner = false;
}
}
}
};
/** Migrate all long-lived buckets from the queue.
* @param now The current time (System.currentTimeMillis()).
* @param force If true, migrate one bucket which isn't necessarily long lived,
* just to free up space. Otherwise we will migrate all long-lived buckets but
* not any others.
* @return True if we migrated any buckets.
* @throws InsufficientSpaceException If there is not enough space to migrate buckets to disk.
*/
private boolean cleanBucketQueue(long now, boolean force) throws InsufficientDiskSpaceException {
boolean shouldContinue = true;
// create a new list to avoid race-conditions
Queue<Migratable> toMigrate = null;
if(logMINOR)
Logger.minor(this, "Starting cleanBucketQueue");
do {
synchronized(ramBucketQueue) {
final WeakReference<Migratable> tmpBucketRef = ramBucketQueue.peek();
if (tmpBucketRef == null)
shouldContinue = false;
else {
Migratable tmpBucket = tmpBucketRef.get();
if (tmpBucket == null) {
ramBucketQueue.remove(tmpBucketRef);
continue; // ugh. this is freed
}
// Don't access the buckets inside the lock, will deadlock.
if (tmpBucket.creationTime() + RAMBUCKET_MAX_AGE > now && !force)
shouldContinue = false;
else {
if (logMINOR)
Logger.minor(this, "The bucket "+tmpBucket+" is " + TimeUtil.formatTime(now - tmpBucket.creationTime())
+ " old: we will force-migrate it to disk.");
ramBucketQueue.remove(tmpBucketRef);
if(toMigrate == null) toMigrate = new LinkedList<Migratable>();
toMigrate.add(tmpBucket);
force = false;
}
}
}
} while(shouldContinue);
if(toMigrate == null) return false;
if(toMigrate.size() > 0) {
if(logMINOR)
Logger.minor(this, "We are going to migrate " + toMigrate.size() + " RAMBuckets");
for(Migratable tmpBucket : toMigrate) {
try {
tmpBucket.migrateToDisk();
} catch (InsufficientDiskSpaceException e) {
throw e;
} catch(IOException e) {
Logger.error(tmpBucket, "An IOE occured while migrating long-lived buckets:" + e.getMessage(), e);
}
}
return true;
}
return false;
}
private final Queue<WeakReference<Migratable>> ramBucketQueue = new LinkedBlockingQueue<WeakReference<Migratable>>();
private RandomAccessBucket _makeFileBucket() throws IOException {
RandomAccessBucket ret = new TempFileBucket(filenameGenerator.makeRandomFilename(), filenameGenerator, true);
// Do we want it to be encrypted?
if(reallyEncrypt) {
ret = new PaddedRandomAccessBucket(ret);
ret = new EncryptedRandomAccessBucket(CRYPT_TYPE, ret, secret);
}
return ret;
}
/** Unlike a TempBucket, the size is fixed, so migrate only happens on the migration thread. */
class TempRandomAccessBuffer extends SwitchableProxyRandomAccessBuffer implements Migratable {
protected boolean hasMigrated = false;
/** If false, there is in-memory storage that needs to be freed. */
private boolean hasFreedRAM = false;
private final long creationTime;
/** Kept in RAM so that finalizer is called on the TempBucket when *both* the
* TempRandomAccessBuffer *and* the TempBucket are no longer reachable, in which case we
* will free from the TempBucket. If this is null, then the TempRAB can free in finalizer.
*/
private final TempBucket original;
/** For debugging leaks if TRACE_BUCKET_LEAKS is enabled */
private final Throwable tracer;
TempRandomAccessBuffer(int size, long time) throws IOException {
super(new ByteArrayRandomAccessBuffer(size), size);
creationTime = time;
hasMigrated = false;
original = null;
if (TRACE_BUCKET_LEAKS)
tracer = new Throwable();
else
tracer = null;
}
public TempRandomAccessBuffer(byte[] initialContents, int offset, int size, long time, boolean readOnly) throws IOException {
super(new ByteArrayRandomAccessBuffer(initialContents, offset, size, readOnly), size);
creationTime = time;
hasMigrated = false;
original = null;
if (TRACE_BUCKET_LEAKS)
tracer = new Throwable();
else
tracer = null;
}
public TempRandomAccessBuffer(LockableRandomAccessBuffer underlying, long creationTime, boolean migrated, TempBucket tempBucket) throws IOException {
super(underlying, underlying.size());
this.creationTime = creationTime;
this.hasMigrated = hasFreedRAM = migrated;
this.original = tempBucket;
if (TRACE_BUCKET_LEAKS)
tracer = new Throwable();
else
tracer = null;
}
@Override
protected LockableRandomAccessBuffer innerMigrate(LockableRandomAccessBuffer underlying) throws IOException {
ByteArrayRandomAccessBuffer b = (ByteArrayRandomAccessBuffer)underlying;
byte[] buf = b.getBuffer();
return diskRAFFactory.makeRAF(buf, 0, (int)size, b.isReadOnly());
}
@Override
public void free() {
if(!super.innerFree()) return;
if(logMINOR) Logger.minor(this, "Freed "+this, new Exception("debug"));
if(original != null) {
// Tell the TempBucket to prevent log spam. Don't call free().
original.onFreed();
}
}
@Override
protected void afterFreeUnderlying() {
// Called when the in-RAM storage has been freed.
synchronized(this) {
if(hasFreedRAM) return;
hasFreedRAM = true;
}
_hasFreed(size);
synchronized(ramBucketQueue) {
ramBucketQueue.remove(getReference());
}
}
private WeakReference<Migratable> weakRef = new WeakReference<Migratable>(this);
public WeakReference<Migratable> getReference() {
return weakRef;
}
@Override
public long creationTime() {
return creationTime;
}
@Override
public boolean migrateToDisk() throws IOException {
synchronized(this) {
if(hasMigrated) return false;
hasMigrated = true;
}
migrate();
return true;
}
public synchronized boolean hasMigrated() {
return hasMigrated;
}
@Override
public void onResume(ClientContext context) {
// Not persistent.
throw new UnsupportedOperationException();
}
@Override
public void storeTo(DataOutputStream dos) throws IOException {
throw new UnsupportedOperationException();
}
@Override
protected void finalize() throws Throwable {
if(original != null) return; // TempBucket's responsibility if there was one.
// If it's been converted to a TempRandomAccessBuffer, finalize() will only be called
// if *neither* object is reachable.
if (!hasBeenFreed()) {
if (TRACE_BUCKET_LEAKS)
Logger.error(this, "TempRandomAccessBuffer not freed, size=" + size() +" : "+this, tracer);
else
Logger.error(this, "TempRandomAccessBuffer not freed, size=" + size() +" : "+this);
free();
}
super.finalize();
}
}
@Override
public LockableRandomAccessBuffer makeRAF(long size) throws IOException {
if(size < 0) throw new IllegalArgumentException();
if(size > Integer.MAX_VALUE) return diskRAFFactory.makeRAF(size);
long now = System.currentTimeMillis();
TempRandomAccessBuffer raf = null;
synchronized(this) {
if((size > 0) && (size <= maxRAMBucketSize) && (bytesInUse < maxRamUsed) && (bytesInUse + size <= maxRamUsed)) {
raf = new TempRandomAccessBuffer((int)size, now);
bytesInUse += size;
}
if(bytesInUse >= maxRamUsed * MAX_USAGE_HIGH && !runningCleaner) {
runningCleaner = true;
executor.execute(cleaner);
}
}
if(raf != null) {
synchronized(ramBucketQueue) {
ramBucketQueue.add(raf.getReference());
}
return raf;
} else {
boolean encrypt;
encrypt = this.reallyEncrypt;
long realSize = size;
long paddedSize = size;
if(encrypt) {
realSize += TempBucketFactory.CRYPT_TYPE.headerLen;
paddedSize = PaddedEphemerallyEncryptedBucket.paddedLength(realSize, PaddedEphemerallyEncryptedBucket.MIN_PADDED_SIZE);
}
LockableRandomAccessBuffer ret = diskRAFFactory.makeRAF(paddedSize);
if(encrypt) {
if(realSize != paddedSize)
ret = new PaddedRandomAccessBuffer(ret, realSize);
try {
ret = new EncryptedRandomAccessBuffer(CRYPT_TYPE, ret, secret, true);
} catch (GeneralSecurityException e) {
Logger.error(this, "Cannot create encrypted tempfile: "+e, e);
}
}
return ret;
}
}
@Override
public LockableRandomAccessBuffer makeRAF(byte[] initialContents, int offset, int size, boolean readOnly)
throws IOException {
if(size < 0) throw new IllegalArgumentException();
long now = System.currentTimeMillis();
TempRandomAccessBuffer raf = null;
synchronized(this) {
if((size > 0) && (size <= maxRAMBucketSize) && (bytesInUse < maxRamUsed) && (bytesInUse + size <= maxRamUsed)) {
raf = new TempRandomAccessBuffer(initialContents, offset, size, now, readOnly);
bytesInUse += size;
}
if(bytesInUse >= maxRamUsed * MAX_USAGE_HIGH && !runningCleaner) {
runningCleaner = true;
executor.execute(cleaner);
}
}
if(raf != null) {
synchronized(ramBucketQueue) {
ramBucketQueue.add(raf.getReference());
}
return raf;
} else {
if(reallyEncrypt) {
// FIXME do the encryption in memory? Test it ...
LockableRandomAccessBuffer ret = makeRAF(size);
ret.pwrite(0, initialContents, offset, size);
if(readOnly) ret = new ReadOnlyRandomAccessBuffer(ret);
return ret;
}
return diskRAFFactory.makeRAF(initialContents, offset, size, readOnly);
}
}
public DiskSpaceCheckingRandomAccessBufferFactory getUnderlyingRAFFactory() {
return diskRAFFactory;
}
}