/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/
package alluxio.client.keyvalue;
import alluxio.Configuration;
import alluxio.Constants;
import alluxio.PropertyKey;
import alluxio.client.AbstractOutStream;
import alluxio.util.io.ByteIOUtils;
import com.google.common.base.Preconditions;
import java.io.IOException;
import javax.annotation.concurrent.NotThreadSafe;
/**
* Writer that implements {@link KeyValuePartitionWriter} using the Alluxio file stream interface to
* generate a single-block key-value file.
* <p>
* A partition file consists of:
* <ul>
* <li>A payload buffer which is an array of (key,value) pairs;</li>
* <li>A index which is a hash table maps each key to the offset in bytes into the payload
* buffer;</li>
* <li>A 4-bytes pointer in the end indicating the offset of the index.</li>
* </ul>
*
*/
@NotThreadSafe
final class BaseKeyValuePartitionWriter implements KeyValuePartitionWriter {
/** Handle to write to the underlying file. */
private final AbstractOutStream mFileOutStream;
/** Number of key-value pairs added. */
private int mKeyCount = 0;
/** Key-value index. */
private Index mIndex;
/** Key-value payload. */
private PayloadWriter mPayloadWriter;
/** Whether this writer is closed. */
private boolean mClosed;
/** Whether this writer is canceled. */
private boolean mCanceled;
/** Maximum size of this partition in bytes. */
private long mMaxSizeBytes;
/**
* Constructs a {@link BaseKeyValuePartitionWriter} given an output stream.
*
* @param fileOutStream output stream to store the key-value file
*/
BaseKeyValuePartitionWriter(AbstractOutStream fileOutStream) {
mFileOutStream = Preconditions.checkNotNull(fileOutStream);
// TODO(binfan): write a header in the file
mPayloadWriter = new BasePayloadWriter(mFileOutStream);
mIndex = LinearProbingIndex.createEmptyIndex();
mClosed = false;
mCanceled = false;
mMaxSizeBytes = Configuration.getBytes(PropertyKey.KEY_VALUE_PARTITION_SIZE_BYTES_MAX);
}
@Override
public void close() throws IOException {
if (mClosed) {
return;
}
if (mCanceled) {
mFileOutStream.cancel();
} else {
build();
mFileOutStream.close();
}
mClosed = true;
}
@Override
public void cancel() throws IOException {
mCanceled = true;
close();
}
@Override
public void put(byte[] key, byte[] value) throws IOException {
Preconditions.checkNotNull(key);
Preconditions.checkNotNull(value);
Preconditions.checkArgument(key.length > 0, "Cannot put an empty key");
Preconditions.checkArgument(value.length > 0, "Cannot put an empty value");
Preconditions.checkState(!mClosed);
mIndex.put(key, value, mPayloadWriter);
mKeyCount++;
}
@Override
public boolean canPut(byte[] key, byte[] value) {
// See BasePayloadWriter.insert()
// TODO(binfan): also take into account the potential index size change
return byteCount() + key.length + value.length
+ Constants.BYTES_IN_INTEGER * 2 <= mMaxSizeBytes;
}
/**
* @return number of keys
*/
@Override
public int keyCount() {
return mKeyCount;
}
/**
* @return number of bytes estimated
*/
public long byteCount() {
Preconditions.checkState(!mClosed);
// last pointer to index
return mFileOutStream.getBytesWritten() + mIndex.byteCount() + Integer.SIZE / Byte.SIZE;
}
private void build() throws IOException {
Preconditions.checkState(!mClosed);
mFileOutStream.flush();
int indexOffset = mFileOutStream.getBytesWritten();
mFileOutStream.write(mIndex.getBytes());
ByteIOUtils.writeInt(mFileOutStream, indexOffset);
}
}