/**
* (C) Copyright IBM Corp. 2010, 2015
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ibm.bi.dml.runtime.controlprogram.caching;
import java.io.DataOutput;
import java.io.IOException;
import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
import com.ibm.bi.dml.runtime.util.LocalFileUtils;
/**
* Wrapper for WriteBuffer byte array per matrix in order to
* support matrix serialization outside global lock.
*
*/
public class ByteBuffer
{
private boolean _serialized;
private boolean _sparse;
private long _size;
protected byte[] _bdata = null; //sparse matrix
protected MatrixBlock _mdata = null; //dense matrix
public ByteBuffer( long size )
{
_size = size;
_serialized = false;
}
/**
*
* @param mb
* @throws IOException
*/
public void serializeMatrix( MatrixBlock mb )
throws IOException
{
boolean sparseSrc = mb.isInSparseFormat(); //current representation
boolean sparseTrgt = mb.evalSparseFormatOnDisk(); //intended target representation
_sparse = sparseTrgt;
try
{
if( _sparse ) //SPARSE/DENSE -> SPARSE
{
//deep serialize (for compression)
if( CacheableData.CACHING_BUFFER_PAGECACHE )
_bdata = PageCache.getPage((int)_size);
if( _bdata==null )
_bdata = new byte[(int)_size];
DataOutput dout = new CacheDataOutput(_bdata);
mb.write(dout);
}
else //SPARSE/DENSE -> DENSE
{
//change representation (if required), incl. free sparse
//(in-memory representation, if dense on disk than if will
//be guaranteed to be dense in memory as well)
if( sparseSrc )
mb.examSparsity();
//shallow serialize
_mdata = mb;
}
}
catch(Exception ex)
{
throw new IOException("Failed to serialize matrix block.", ex);
}
_serialized = true;
}
/**
*
* @return
* @throws IOException
*/
public MatrixBlock deserializeMatrix()
throws IOException
{
MatrixBlock ret = null;
if( _sparse )
{
//ByteArrayInputStream bis = new ByteArrayInputStream(_bdata);
//DataInputStream din = new DataInputStream(bis);
CacheDataInput din = new CacheDataInput(_bdata);
ret = new MatrixBlock();
ret.readFields(din);
}
else
{
ret = _mdata;
}
return ret;
}
/**
*
* @param fname
* @throws IOException
*/
public void evictBuffer( String fname )
throws IOException
{
if( _sparse )
{
//write out byte serialized array
LocalFileUtils.writeByteArrayToLocal(fname, _bdata);
}
else
{
//serialize matrix to output stream
LocalFileUtils.writeMatrixBlockToLocal(fname, _mdata);
}
}
/**
* Returns the buffer size in bytes.
*
* @return
*/
public long getSize()
{
return _size;
}
/**
*
* @return
*/
public boolean isInSparseFormat()
{
return _sparse;
}
public void freeMemory()
{
//clear strong references to buffer/matrix
if( _sparse )
{
if( CacheableData.CACHING_BUFFER_PAGECACHE )
PageCache.putPage(_bdata);
_bdata = null;
}
else
{
_mdata = null;
}
}
/**
*
*/
public void checkSerialized()
{
if( _serialized )
return;
while( !_serialized )
{
try{Thread.sleep(1);} catch(Exception e) {}
}
}
/**
* Determines if byte buffer can hold the given size given this specific matrix block.
* This call is consistent with 'serializeMatrix' and allows for internal optimization
* according to dense/sparse representation.
*
* @param size
* @param mb
* @return
*/
public static boolean isValidCapacity( long size, MatrixBlock mb )
{
boolean sparseTrgt = mb.evalSparseFormatOnDisk(); //intended target representation
if( sparseTrgt ) //SPARSE
{
// since sparse matrix blocks are serialized into a byte representation
// the buffer buffer can hold at most 2GB in size
return ( size <= Integer.MAX_VALUE );
}
else //DENSE
{
// since for dense matrix blocks we use a shallow serialize (strong reference),
// the byte buffer can hold any size (currently upper bounded by 16GB)
return true;
}
}
}