/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.sysml.runtime.io; import java.io.IOException; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.matrix.data.MatrixBlock; /** * Base class for all format-specific matrix writers. Every writer is required to implement the basic * write functionality but might provide additional custom functionality. Any non-default parameters * (e.g., CSV read properties) should be passed into custom constructors. There is also a factory * for creating format-specific writers. * */ public abstract class MatrixWriter { public abstract void writeMatrixToHDFS( MatrixBlock src, String fname, long rlen, long clen, int brlen, int bclen, long nnz ) throws IOException, DMLRuntimeException; /** * Writes a minimal entry to represent an empty matrix on hdfs. * * @param fname file name * @param rlen number of rows * @param clen number of columns * @param brlen number of rows in block * @param bclen number of columns in block * @throws IOException if IOException occurs * @throws DMLRuntimeException if DMLRuntimeException occurs */ public abstract void writeEmptyMatrixToHDFS( String fname, long rlen, long clen, int brlen, int bclen ) throws IOException, DMLRuntimeException; public static MatrixBlock[] createMatrixBlocksForReuse( long rlen, long clen, int brlen, int bclen, boolean sparse, long nonZeros ) throws DMLRuntimeException { MatrixBlock[] blocks = new MatrixBlock[4]; double sparsity = ((double)nonZeros)/(rlen*clen); long estNNZ = -1; //full block if( rlen >= brlen && clen >= bclen ) { estNNZ = (long) (brlen*bclen*sparsity); blocks[0] = new MatrixBlock( brlen, bclen, sparse, (int)estNNZ ); } //partial col block if( rlen >= brlen && clen%bclen!=0 ) { estNNZ = (long) (brlen*(clen%bclen)*sparsity); blocks[1] = new MatrixBlock( brlen, (int)(clen%bclen), sparse, (int)estNNZ ); } //partial row block if( rlen%brlen!=0 && clen>=bclen ) { estNNZ = (long) ((rlen%brlen)*bclen*sparsity); blocks[2] = new MatrixBlock( (int)(rlen%brlen), bclen, sparse, (int)estNNZ ); } //partial row/col block if( rlen%brlen!=0 && clen%bclen!=0 ) { estNNZ = (long) ((rlen%brlen)*(clen%bclen)*sparsity); blocks[3] = new MatrixBlock( (int)(rlen%brlen), (int)(clen%bclen), sparse, (int)estNNZ ); } //space allocation for( MatrixBlock b : blocks ) if( b != null ) if( !sparse ) b.allocateDenseBlockUnsafe(b.getNumRows(), b.getNumColumns()); //NOTE: no preallocation for sparse (preallocate sparserows with estnnz) in order to reduce memory footprint return blocks; } public static MatrixBlock getMatrixBlockForReuse( MatrixBlock[] blocks, int rows, int cols, int brlen, int bclen ) { int index = -1; if( rows==brlen && cols==bclen ) index = 0; else if( rows==brlen && cols<bclen ) index = 1; else if( rows<brlen && cols==bclen ) index = 2; else //if( rows<brlen && cols<bclen ) index = 3; return blocks[ index ]; } }