/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/
package alluxio.client.block.stream;
import alluxio.Configuration;
import alluxio.PropertyKey;
import alluxio.client.file.FileSystemContext;
import alluxio.exception.status.AlluxioStatusException;
import alluxio.exception.status.CanceledException;
import alluxio.exception.status.DeadlineExceededException;
import alluxio.network.protocol.RPCProtoMessage;
import alluxio.network.protocol.databuffer.DataBuffer;
import alluxio.network.protocol.databuffer.DataNettyBufferV2;
import alluxio.proto.dataserver.Protocol;
import alluxio.proto.status.Status.PStatus;
import alluxio.util.network.NettyUtils;
import alluxio.util.proto.ProtoMessage;
import alluxio.wire.WorkerNetAddress;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import javax.annotation.concurrent.NotThreadSafe;
/**
* A netty packet reader that streams a region from a netty data server.
*
* Protocol:
* 1. The client sends a read request (id, offset, length).
* 2. Once the server receives the request, it streams packets to the client. The streaming pauses
* if the server's buffer is full and resumes if the buffer is not full.
* 3. The client reads packets from the stream. Reading pauses if the client buffer is full and
* resumes if the buffer is not full. If the client can keep up with network speed, the buffer
* should have at most one packet.
* 4. The client stops reading if it receives an empty packet which signifies the end of the stream.
* 5. The client can cancel the read request at anytime. The cancel request is ignored by the
* server if everything has been sent to channel.
* 6. If the client wants to reuse the channel, the client must read all the packets in the channel
* before releasing the channel to the channel pool.
* 7. To make it simple to handle errors, the channel is closed if any error occurs.
*/
@NotThreadSafe
public final class NettyPacketReader implements PacketReader {
private static final Logger LOG = LoggerFactory.getLogger(NettyPacketReader.class);
private static final int MAX_PACKETS_IN_FLIGHT =
Configuration.getInt(PropertyKey.USER_NETWORK_NETTY_READER_BUFFER_SIZE_PACKETS);
private static final long READ_TIMEOUT_MS =
Configuration.getLong(PropertyKey.USER_NETWORK_NETTY_TIMEOUT_MS);
/** Special packet that indicates an exception is caught. */
private static final ByteBuf THROWABLE = Unpooled.buffer(0);
/** Special packet that indicates the EOF is reached or the stream is cancelled. */
private static final ByteBuf EOF_OR_CANCELLED = Unpooled.buffer(0);
private final FileSystemContext mContext;
private final Channel mChannel;
private final Protocol.RequestType mRequestType;
private final WorkerNetAddress mAddress;
private final long mId;
private final long mStart;
private final long mBytesToRead;
private final boolean mNoCache;
private final long mPacketSize;
/**
* This queue contains buffers read from netty. Its length is bounded by MAX_PACKETS_IN_FLIGHT.
* Only the netty I/O thread can push to the queue. Only the client thread can poll from the
* queue.
*/
private final BlockingQueue<ByteBuf> mPackets = new LinkedBlockingQueue<>();
/**
* The exception caught when reading packets from the netty channel. This is only updated
* by the netty I/O thread. The client thread only reads it after THROWABLE is found in
* mPackets queue.
*/
private volatile Throwable mPacketReaderException;
/**
* The next pos to read. This is only updated by the client thread (not touched by the netty
* I/O thread).
*/
private long mPosToRead;
/**
* This is true only when an empty packet (EOF or CANCELLED) is received. This is only updated
* by the client thread (not touched by the netty I/O thread).
*/
private boolean mDone = false;
private boolean mClosed = false;
/**
* Creates an instance of {@link NettyPacketReader}. If this is used to read a block remotely, it
* requires the block to be locked beforehand and the lock ID is passed to this class.
*
* @param context the file system context
* @param address the netty data server address
* @param id the block ID or UFS file ID
* @param offset the offset
* @param len the length to read
* @param lockId the lock ID
* @param sessionId the session ID
* @param noCache do not cache the block to the Alluxio worker if read from UFS when this is set
* @param type the request type (block or UFS file)
* @param packetSize the packet size
*/
private NettyPacketReader(FileSystemContext context, WorkerNetAddress address, long id,
long offset, long len, long lockId, long sessionId, boolean noCache,
Protocol.RequestType type, long packetSize) throws IOException {
Preconditions.checkArgument(offset >= 0 && len > 0 && packetSize > 0);
mContext = context;
mAddress = address;
mId = id;
mStart = offset;
mPosToRead = offset;
mBytesToRead = len;
mRequestType = type;
mNoCache = noCache;
mPacketSize = packetSize;
mChannel = mContext.acquireNettyChannel(address);
mChannel.pipeline().addLast(new PacketReadHandler());
Protocol.ReadRequest readRequest =
Protocol.ReadRequest.newBuilder().setId(id).setOffset(offset).setLength(len)
.setLockId(lockId).setSessionId(sessionId).setType(type).setNoCache(noCache)
.setPacketSize(packetSize).build();
mChannel.writeAndFlush(new RPCProtoMessage(new ProtoMessage(readRequest)))
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
}
@Override
public long pos() {
return mPosToRead;
}
@Override
public DataBuffer readPacket() throws IOException {
Preconditions.checkState(!mClosed, "PacketReader is closed while reading packets.");
ByteBuf buf;
// TODO(peis): Have a better criteria to resume so that we can have fewer state changes.
if (!tooManyPacketsPending()) {
NettyUtils.enableAutoRead(mChannel);
}
try {
buf = mPackets.poll(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new CanceledException(e);
}
if (buf == null) {
throw new DeadlineExceededException(
String.format("Timeout to read %d from %s.", mId, mChannel.toString()));
}
if (buf == THROWABLE) {
Preconditions.checkNotNull(mPacketReaderException, "mPacketReaderException");
Throwables.propagateIfPossible(mPacketReaderException, IOException.class);
throw AlluxioStatusException.fromCheckedException(mPacketReaderException);
}
if (buf == EOF_OR_CANCELLED) {
mDone = true;
return null;
}
mPosToRead += buf.readableBytes();
Preconditions.checkState(mPosToRead - mStart <= mBytesToRead);
return new DataNettyBufferV2(buf);
}
@Override
public void close() {
if (mClosed) {
return;
}
try {
if (mDone) {
return;
}
if (!mChannel.isOpen()) {
return;
}
if (remaining() > 0) {
Protocol.ReadRequest cancelRequest =
Protocol.ReadRequest.newBuilder().setId(mId).setCancel(true).setType(mRequestType)
.setNoCache(mNoCache).build();
mChannel.writeAndFlush(new RPCProtoMessage(new ProtoMessage(cancelRequest)))
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
}
try {
readAndDiscardAll();
} catch (IOException e) {
LOG.warn("Failed to close the NettyBlockReader (block: {}, address: {}) with exception {}.",
mId, mAddress, e.getMessage());
mChannel.close();
return;
}
} finally {
if (mChannel.isOpen()) {
mChannel.pipeline().removeLast();
// Make sure "autoread" is on before releasing the channel.
NettyUtils.enableAutoRead(mChannel);
}
mContext.releaseNettyChannel(mAddress, mChannel);
mClosed = true;
}
}
/**
* Reads and discards everything read from the channel until it reaches end of the stream.
*/
private void readAndDiscardAll() throws IOException {
DataBuffer buf;
do {
buf = readPacket();
if (buf != null) {
buf.release();
}
// A null packet indicates the end of the stream.
} while (buf != null);
}
/**
* @return bytes remaining
*/
private long remaining() {
return mStart + mBytesToRead - mPosToRead;
}
/**
* @return true if there are too many packets pending
*/
private boolean tooManyPacketsPending() {
return mPackets.size() >= MAX_PACKETS_IN_FLIGHT;
}
/**
* The netty handler that reads packets from the channel.
*/
private class PacketReadHandler extends ChannelInboundHandlerAdapter {
/**
* Default constructor.
*/
PacketReadHandler() {}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws IOException {
// Precondition check is not used here to avoid calling msg.getClass().getCanonicalName()
// all the time.
if (!acceptMessage(msg)) {
throw new IllegalStateException(String
.format("Incorrect response type %s, %s.", msg.getClass().getCanonicalName(), msg));
}
RPCProtoMessage response = (RPCProtoMessage) msg;
// Canceled is considered a valid status and handled in the reader. We avoid creating a
// CanceledException as an optimization.
if (response.getMessage().asResponse().getStatus() != PStatus.CANCELED) {
response.unwrapException();
}
DataBuffer dataBuffer = response.getPayloadDataBuffer();
ByteBuf buf;
if (dataBuffer == null) {
buf = EOF_OR_CANCELLED;
} else {
Preconditions.checkState(
dataBuffer.getLength() > 0 && (dataBuffer.getNettyOutput() instanceof ByteBuf));
buf = (ByteBuf) dataBuffer.getNettyOutput();
}
if (tooManyPacketsPending()) {
NettyUtils.disableAutoRead(ctx.channel());
}
mPackets.offer(buf);
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
LOG.error("Exception caught while reading from {}.", mId, cause);
// NOTE: The netty I/O thread associated with mChannel is the only thread that can update
// mPacketReaderException and push to mPackets. So it is safe to do the following without
// synchronization.
// Make sure to set mPacketReaderException before pushing THROWABLE to mPackets.
if (mPacketReaderException == null) {
mPacketReaderException = cause;
mPackets.offer(THROWABLE);
}
ctx.close();
}
@Override
public void channelUnregistered(ChannelHandlerContext ctx) {
LOG.warn("Channel {} is closed while reading from {}.", mChannel, mId);
// NOTE: The netty I/O thread associated with mChannel is the only thread that can update
// mPacketReaderException and push to mPackets. So it is safe to do the following without
// synchronization.
// Make sure to set mPacketReaderException before pushing THROWABLE to mPackets.
if (mPacketReaderException == null) {
mPacketReaderException =
new IOException(String.format("Channel %s is closed.", mChannel.toString()));
mPackets.offer(THROWABLE);
}
ctx.fireChannelUnregistered();
}
/**
* @param msg the message received
* @return true if this message should be processed
*/
private boolean acceptMessage(Object msg) {
if (msg instanceof RPCProtoMessage) {
return ((RPCProtoMessage) msg).getMessage().isResponse();
}
return false;
}
}
/**
* Factory class to create {@link NettyPacketReader}s.
*/
public static class Factory implements PacketReader.Factory {
private final FileSystemContext mContext;
private final WorkerNetAddress mAddress;
private final long mId;
private final long mLockId;
private final long mSessionId;
private final boolean mNoCache;
private final Protocol.RequestType mRequestType;
private final long mPacketSize;
/**
* Creates an instance of {@link NettyPacketReader.Factory} for block reads.
*
* @param context the file system context
* @param address the worker address
* @param id the block ID or UFS ID
* @param lockId the lock ID
* @param sessionId the session ID
* @param noCache if set, the block won't be cached in Alluxio if the block is a UFS block
* @param type the request type
* @param packetSize the packet size
*/
public Factory(FileSystemContext context, WorkerNetAddress address, long id, long lockId,
long sessionId, boolean noCache, Protocol.RequestType type, long packetSize) {
mContext = context;
mAddress = address;
mId = id;
mLockId = lockId;
mSessionId = sessionId;
mNoCache = noCache;
mRequestType = type;
mPacketSize = packetSize;
}
@Override
public PacketReader create(long offset, long len) throws IOException {
return new NettyPacketReader(mContext, mAddress, mId, offset, len, mLockId, mSessionId,
mNoCache, mRequestType, mPacketSize);
}
@Override
public boolean isShortCircuit() {
return false;
}
}
}