/*
* Copyright (c) 2002-2009 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.nioneo.store;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.FileChannel;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.neo4j.kernel.impl.util.StringLogger;
/**
* An abstract representation of a dynamic store. The difference between a
* normal {@link AbstractStore} and a <CODE>AbstractDynamicStore</CODE> is
* that the size of a record/entry can be dynamic.
* <p>
* Instead of a fixed record this class uses blocks to store a record. If a
* record size is greater than the block size the record will use one or more
* blocks to store its data.
* <p>
* A dynamic store don't have a {@link IdGenerator} because the position of a
* record can't be calculated just by knowing the id. Instead one should use a
* {@link AbstractStore} and store the start block of the record located in the
* dynamic store. Note: This class makes use of an id generator internally for
* managing free and non free blocks.
* <p>
* Note, the first block of a dynamic store is reserved and contains information
* about the store.
*/
public abstract class AbstractDynamicStore extends CommonAbstractStore
{
/**
* Creates a new empty store. A factory method returning an implementation
* should make use of this method to initialize an empty store. Block size
* must be greater than zero. Not that the first block will be marked as
* reserved (contains info about the block size). There will be an overhead
* for each block of <CODE>13</CODE> bytes.
* <p>
* This method will create a empty store with descriptor returned by the
* {@link #getTypeAndVersionDescriptor()}. The internal id generator used
* by this store will also be created.
*
* @param fileName
* The file name of the store that will be created
* @param blockSize
* The number of bytes for each block
* @param typeAndVersionDescriptor
* The type and version descriptor that identifies this store
*
* @throws IOException
* If fileName is null or if file exists or illegal block size
*/
protected static void createEmptyStore( String fileName, int baseBlockSize,
String typeAndVersionDescriptor )
{
int blockSize = baseBlockSize;
// sanity checks
if ( fileName == null )
{
throw new IllegalArgumentException( "Null filename" );
}
File file = new File( fileName );
if ( file.exists() )
{
throw new IllegalStateException( "Can't create store[" + fileName
+ "], file already exists" );
}
if ( blockSize < 1 )
{
throw new IllegalArgumentException( "Illegal block size["
+ blockSize + "]" );
}
blockSize += 13; // in_use(1)+length(4)+prev_block(4)+next_block(4)
// write the header
try
{
FileChannel channel = new FileOutputStream( fileName ).getChannel();
int endHeaderSize = blockSize
+ typeAndVersionDescriptor.getBytes().length;
ByteBuffer buffer = ByteBuffer.allocate( endHeaderSize );
buffer.putInt( blockSize );
buffer.position( endHeaderSize - typeAndVersionDescriptor.length() );
buffer.put( typeAndVersionDescriptor.getBytes() ).flip();
channel.write( buffer );
channel.force( false );
channel.close();
}
catch ( IOException e )
{
throw new UnderlyingStorageException( "Unable to create store "
+ fileName, e );
}
IdGeneratorImpl.createGenerator( fileName + ".id" );
IdGenerator idGenerator = new IdGeneratorImpl( fileName + ".id", 1 );
idGenerator.nextId(); // reserv first for blockSize
idGenerator.close();
}
private int blockSize;
public AbstractDynamicStore( String fileName, Map<?,?> config )
{
super( fileName, config );
}
public AbstractDynamicStore( String fileName )
{
super( fileName );
}
/**
* Loads this store validating version and id generator. Also the block size
* is loaded (contained in first block)
*/
protected void loadStorage()
{
try
{
long fileSize = getFileChannel().size();
String expectedVersion = getTypeAndVersionDescriptor();
byte version[] = new byte[expectedVersion.getBytes().length];
ByteBuffer buffer = ByteBuffer.wrap( version );
getFileChannel().position( fileSize - version.length );
getFileChannel().read( buffer );
buffer = ByteBuffer.allocate( 4 );
getFileChannel().position( 0 );
getFileChannel().read( buffer );
buffer.flip();
blockSize = buffer.getInt();
if ( blockSize <= 0 )
{
throw new InvalidRecordException( "Illegal block size: " +
blockSize + " in " + getStorageFileName() );
}
if ( !expectedVersion.equals( new String( version ) ) )
{
if ( !versionFound( new String( version ) ) && !isReadOnly() )
{
setStoreNotOk();
}
}
if ( (fileSize - version.length) % blockSize != 0 && !isReadOnly() )
{
setStoreNotOk();
}
if ( getStoreOk() && !isReadOnly() )
{
getFileChannel().truncate( fileSize - version.length );
}
}
catch ( IOException e )
{
throw new UnderlyingStorageException( "Unable to load storage "
+ getStorageFileName(), e );
}
try
{
if ( !isReadOnly() || isBackupSlave() )
{
openIdGenerator();
}
else
{
openReadOnlyIdGenerator( getBlockSize() );
}
}
catch ( InvalidIdGeneratorException e )
{
setStoreNotOk();
}
finally
{
if ( !getStoreOk() )
{
if ( getConfig() != null )
{
String storeDir = (String) getConfig().get( "store_dir" );
StringLogger msgLog = StringLogger.getLogger(
storeDir + "/messages.log" );
msgLog.logMessage( getStorageFileName() + " non clean shutdown detected" );
}
}
}
setWindowPool( new PersistenceWindowPool( getStorageFileName(),
getBlockSize(), getFileChannel(), getMappedMem(),
getIfMemoryMapped(), isReadOnly() && !isBackupSlave() ) );
}
/**
* Returns the byte size of each block for this dynamic store
*
* @return The block size of this store
*/
public int getBlockSize()
{
return blockSize;
}
/**
* Returns next free block.
*
* @return The next free block
* @throws IOException
* If capacity exceeded or closed id generator
*/
public int nextBlockId()
{
return nextId();
}
/**
* Makes a previously used block available again.
*
* @param blockId
* The id of the block to free
* @throws IOException
* If id generator closed or illegal block id
*/
public void freeBlockId( int blockId )
{
freeId( blockId );
}
// in_use(byte)+prev_block(int)+nr_of_bytes(int)+next_block(int)
private static final int BLOCK_HEADER_SIZE = 1 + 4 + 4 + 4;
public void updateRecord( DynamicRecord record )
{
int blockId = record.getId();
PersistenceWindow window = acquireWindow( blockId, OperationType.WRITE );
try
{
Buffer buffer = window.getOffsettedBuffer( blockId );
if ( record.inUse() )
{
assert record.getId() != record.getPrevBlock();
buffer.put( Record.IN_USE.byteValue() ).putInt(
record.getPrevBlock() ).putInt( record.getLength() )
.putInt( record.getNextBlock() );
if ( !record.isLight() )
{
if ( !record.isCharData() )
{
buffer.put( record.getData() );
}
else
{
buffer.put( record.getDataAsChar() );
}
}
}
else
{
buffer.put( Record.NOT_IN_USE.byteValue() );
if ( !isInRecoveryMode() )
{
freeBlockId( blockId );
}
}
}
finally
{
releaseWindow( window );
}
}
public Collection<DynamicRecord> allocateRecords( int startBlock,
byte src[] )
{
assert getFileChannel() != null : "Store closed, null file channel";
assert src != null : "Null src argument";
List<DynamicRecord> recordList = new LinkedList<DynamicRecord>();
int nextBlock = startBlock;
int prevBlock = Record.NO_PREV_BLOCK.intValue();
int srcOffset = 0;
int dataSize = getBlockSize() - BLOCK_HEADER_SIZE;
do
{
DynamicRecord record = new DynamicRecord( nextBlock );
record.setCreated();
record.setInUse( true );
assert prevBlock != nextBlock;
record.setPrevBlock( prevBlock );
if ( src.length - srcOffset > dataSize )
{
byte data[] = new byte[dataSize];
System.arraycopy( src, srcOffset, data, 0, dataSize );
record.setData( data );
prevBlock = nextBlock;
nextBlock = nextBlockId();
record.setNextBlock( nextBlock );
srcOffset += dataSize;
}
else
{
byte data[] = new byte[src.length - srcOffset];
System.arraycopy( src, srcOffset, data, 0, data.length );
record.setData( data );
nextBlock = Record.NO_NEXT_BLOCK.intValue();
record.setNextBlock( nextBlock );
}
recordList.add( record );
}
while ( nextBlock != Record.NO_NEXT_BLOCK.intValue() );
return recordList;
}
public Collection<DynamicRecord> allocateRecords( int startBlock,
char src[] )
{
assert getFileChannel() != null : "Store closed, null file channel";
assert src != null : "Null src argument";
List<DynamicRecord> recordList = new LinkedList<DynamicRecord>();
int nextBlock = startBlock;
int prevBlock = Record.NO_PREV_BLOCK.intValue();
int srcOffset = 0;
int dataSize = getBlockSize() - BLOCK_HEADER_SIZE;
do
{
DynamicRecord record = new DynamicRecord( nextBlock );
record.setCreated();
record.setInUse( true );
assert prevBlock != nextBlock;
record.setPrevBlock( prevBlock );
if ( (src.length - srcOffset) * 2 > dataSize )
{
byte data[] = new byte[dataSize];
CharBuffer charBuf = ByteBuffer.wrap( data ).asCharBuffer();
charBuf.put( src, srcOffset, dataSize / 2 );
record.setData( data );
prevBlock = nextBlock;
nextBlock = nextBlockId();
record.setNextBlock( nextBlock );
srcOffset += dataSize / 2;
}
else
{
if ( srcOffset == 0 )
{
record.setCharData( src );
}
else
{
byte data[] = new byte[(src.length - srcOffset) * 2];
CharBuffer charBuf = ByteBuffer.wrap( data ).asCharBuffer();
charBuf.put( src, srcOffset, src.length - srcOffset );
record.setData( data );
}
nextBlock = Record.NO_NEXT_BLOCK.intValue();
record.setNextBlock( nextBlock );
}
recordList.add( record );
}
while ( nextBlock != Record.NO_NEXT_BLOCK.intValue() );
return recordList;
}
public Collection<DynamicRecord> getLightRecords( int startBlockId )
{
List<DynamicRecord> recordList = new LinkedList<DynamicRecord>();
int blockId = startBlockId;
while ( blockId != Record.NO_NEXT_BLOCK.intValue() )
{
PersistenceWindow window = acquireWindow( blockId,
OperationType.READ );
try
{
DynamicRecord record = getLightRecord(
blockId, window );
recordList.add( record );
blockId = record.getNextBlock();
}
finally
{
releaseWindow( window );
}
}
return recordList;
}
public void makeHeavy( DynamicRecord record )
{
int blockId = record.getId();
PersistenceWindow window = acquireWindow( blockId, OperationType.READ );
try
{
Buffer buf = window.getBuffer();
// NOTE: skip of header in offset
int offset = (int) ((blockId & 0xFFFFFFFFL) -
buf.position()) * getBlockSize() + BLOCK_HEADER_SIZE;
buf.setOffset( offset );
byte bytes[] = new byte[record.getLength()];
buf.get( bytes );
record.setData( bytes );
}
finally
{
releaseWindow( window );
}
}
private DynamicRecord getLightRecord( int blockId, PersistenceWindow window )
{
DynamicRecord record = new DynamicRecord( blockId );
Buffer buffer = window.getOffsettedBuffer( blockId );
byte inUse = buffer.get();
if ( inUse != Record.IN_USE.byteValue() )
{
throw new InvalidRecordException( "Block not inUse[" + inUse
+ "] blockId[" + blockId + "]" );
}
record.setInUse( true );
int prevBlock = buffer.getInt();
record.setPrevBlock( prevBlock );
int dataSize = getBlockSize() - BLOCK_HEADER_SIZE;
int nrOfBytes = buffer.getInt();
int nextBlock = buffer.getInt();
if ( nextBlock != Record.NO_NEXT_BLOCK.intValue()
&& nrOfBytes < dataSize || nrOfBytes > dataSize )
{
throw new InvalidRecordException( "Next block set[" + nextBlock
+ "] current block illegal size[" + nrOfBytes + "/" + dataSize
+ "]" );
}
record.setLength( nrOfBytes );
record.setNextBlock( nextBlock );
record.setIsLight( true );
return record;
}
private DynamicRecord getRecord( int blockId, PersistenceWindow window )
{
DynamicRecord record = new DynamicRecord( blockId );
Buffer buffer = window.getOffsettedBuffer( blockId );
byte inUse = buffer.get();
if ( inUse != Record.IN_USE.byteValue() )
{
throw new InvalidRecordException( "Not in use [" + inUse
+ "] blockId[" + blockId + "]" );
}
record.setInUse( true );
int prevBlock = buffer.getInt();
record.setPrevBlock( prevBlock );
int dataSize = getBlockSize() - BLOCK_HEADER_SIZE;
int nrOfBytes = buffer.getInt();
int nextBlock = buffer.getInt();
if ( nextBlock != Record.NO_NEXT_BLOCK.intValue()
&& nrOfBytes < dataSize || nrOfBytes > dataSize )
{
throw new InvalidRecordException( "Next block set[" + nextBlock
+ "] current block illegal size[" + nrOfBytes + "/" + dataSize
+ "]" );
}
record.setLength( nrOfBytes );
record.setNextBlock( nextBlock );
byte byteArrayElement[] = new byte[nrOfBytes];
buffer.get( byteArrayElement );
record.setData( byteArrayElement );
return record;
}
public Collection<DynamicRecord> getRecords( int startBlockId )
{
List<DynamicRecord> recordList = new LinkedList<DynamicRecord>();
int blockId = startBlockId;
while ( blockId != Record.NO_NEXT_BLOCK.intValue() )
{
PersistenceWindow window = acquireWindow( blockId,
OperationType.READ );
try
{
DynamicRecord record = getRecord( blockId, window );
recordList.add( record );
blockId = record.getNextBlock();
}
finally
{
releaseWindow( window );
}
}
return recordList;
}
/**
* Reads a <CODE>byte array</CODE> stored in this dynamic store using
* <CODE>blockId</CODE> as start block.
*
* @param blockId
* The starting block id
* @return The <CODE>byte array</CODE> stored
* @throws IOException
* If unable to read the data
*/
protected byte[] get( int blockId )
{
LinkedList<byte[]> byteArrayList = new LinkedList<byte[]>();
PersistenceWindow window = acquireWindow( blockId, OperationType.READ );
try
{
Buffer buffer = window.getOffsettedBuffer( blockId );
byte inUse = buffer.get();
if ( inUse != Record.IN_USE.byteValue() )
{
throw new InvalidRecordException( "Not in use [" + inUse
+ "] blockId[" + blockId + "]" );
}
int prevBlock = buffer.getInt();
if ( prevBlock != Record.NO_PREV_BLOCK.intValue() )
{
throw new InvalidRecordException(
"Start[" + blockId + "] block has previous[" + prevBlock +
"] block set" );
}
int nextBlock = blockId;
int dataSize = getBlockSize() - BLOCK_HEADER_SIZE;
do
{
int nrOfBytes = buffer.getInt();
prevBlock = nextBlock;
nextBlock = buffer.getInt();
if ( nextBlock != Record.NO_NEXT_BLOCK.intValue()
&& nrOfBytes < dataSize || nrOfBytes > dataSize )
{
throw new InvalidRecordException( "Next block set["
+ nextBlock + "] current block illegal size["
+ nrOfBytes + "/" + dataSize + "]" );
}
byte byteArrayElement[] = new byte[nrOfBytes];
buffer.get( byteArrayElement );
byteArrayList.add( byteArrayElement );
if ( nextBlock != Record.NO_NEXT_BLOCK.intValue() )
{
releaseWindow( window );
window = acquireWindow( nextBlock, OperationType.READ );
buffer = window.getOffsettedBuffer( nextBlock );
inUse = buffer.get();
if ( inUse != Record.IN_USE.byteValue() )
{
throw new InvalidRecordException( "Next block["
+ nextBlock + "] not in use [" + inUse + "]" );
}
if ( buffer.getInt() != prevBlock )
{
throw new InvalidRecordException(
"Previous[" + prevBlock + "] block don't match" );
}
}
}
while ( nextBlock != Record.NO_NEXT_BLOCK.intValue() );
}
finally
{
releaseWindow( window );
}
int totalSize = 0;
Iterator<byte[]> itr = byteArrayList.iterator();
while ( itr.hasNext() )
{
totalSize += itr.next().length;
}
byte allBytes[] = new byte[totalSize];
itr = byteArrayList.iterator();
int index = 0;
while ( itr.hasNext() )
{
byte currentArray[] = itr.next();
System.arraycopy( currentArray, 0, allBytes, index,
currentArray.length );
index += currentArray.length;
}
return allBytes;
}
private int findHighIdBackwards() throws IOException
{
FileChannel fileChannel = getFileChannel();
int recordSize = getBlockSize();
long fileSize = fileChannel.size();
long highId = fileSize / recordSize;
ByteBuffer byteBuffer = ByteBuffer.allocate( 1 );
for ( long i = highId; i > 0; i-- )
{
fileChannel.position( i * recordSize );
if ( fileChannel.read( byteBuffer ) > 0 )
{
byteBuffer.flip();
byte inUse = byteBuffer.get();
byteBuffer.clear();
if ( inUse != 0 )
{
return (int) i;
}
}
}
return 0;
}
/**
* Rebuilds the internal id generator keeping track of what blocks are free
* or taken.
*
* @throws IOException
* If unable to rebuild the id generator
*/
protected void rebuildIdGenerator()
{
if ( getBlockSize() <= 0 )
{
throw new InvalidRecordException( "Illegal blockSize: " +
getBlockSize() );
}
logger.fine( "Rebuilding id generator for[" + getStorageFileName()
+ "] ..." );
closeIdGenerator();
File file = new File( getStorageFileName() + ".id" );
if ( file.exists() )
{
boolean success = file.delete();
assert success;
}
IdGeneratorImpl.createGenerator( getStorageFileName() + ".id" );
openIdGenerator();
nextBlockId(); // reserved first block containing blockSize
FileChannel fileChannel = getFileChannel();
int highId = 0;
long defraggedCount = 0;
try
{
long fileSize = fileChannel.size();
boolean fullRebuild = true;
if ( getConfig() != null )
{
String mode = (String)
getConfig().get( "rebuild_idgenerators_fast" );
if ( mode != null && mode.toLowerCase().equals( "true" ) )
{
fullRebuild = false;
highId = findHighIdBackwards();
}
}
ByteBuffer byteBuffer = ByteBuffer.wrap( new byte[1] );
LinkedList<Integer> freeIdList = new LinkedList<Integer>();
if ( fullRebuild )
{
for ( long i = 1; i * getBlockSize() < fileSize; i++ )
{
fileChannel.position( i * getBlockSize() );
fileChannel.read( byteBuffer );
byteBuffer.flip();
byte inUse = byteBuffer.get();
byteBuffer.flip();
nextBlockId();
if ( inUse == Record.NOT_IN_USE.byteValue() )
{
freeIdList.add( (int) i );
}
else
{
highId = (int) i;
while ( !freeIdList.isEmpty() )
{
freeBlockId( freeIdList.removeFirst() );
defraggedCount++;
}
}
}
}
}
catch ( IOException e )
{
throw new UnderlyingStorageException(
"Unable to rebuild id generator " + getStorageFileName(), e );
}
setHighId( highId + 1 );
logger.fine( "[" + getStorageFileName() + "] high id=" + getHighId()
+ " (defragged=" + defraggedCount + ")" );
if ( getConfig() != null )
{
String storeDir = (String) getConfig().get( "store_dir" );
StringLogger msgLog = StringLogger.getLogger(
storeDir + "/messages.log" );
msgLog.logMessage( getStorageFileName() + " rebuild id generator, highId=" + getHighId() +
" defragged count=" + defraggedCount );
}
closeIdGenerator();
openIdGenerator();
}
protected void updateHighId()
{
try
{
long highId = getFileChannel().size() / getBlockSize();
super.setHighId( highId );
}
catch ( IOException e )
{
throw new UnderlyingStorageException( e );
}
}
}