/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.runtime.io.gates;
import java.io.IOException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicReference;
import eu.stratosphere.nephele.deployment.ChannelDeploymentDescriptor;
import eu.stratosphere.nephele.deployment.GateDeploymentDescriptor;
import eu.stratosphere.runtime.io.Buffer;
import eu.stratosphere.runtime.io.network.bufferprovider.BufferAvailabilityListener;
import eu.stratosphere.runtime.io.network.bufferprovider.BufferProvider;
import eu.stratosphere.runtime.io.network.bufferprovider.GlobalBufferPool;
import eu.stratosphere.runtime.io.network.bufferprovider.LocalBufferPool;
import eu.stratosphere.runtime.io.network.bufferprovider.LocalBufferPoolOwner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.stratosphere.core.io.IOReadableWritable;
import eu.stratosphere.nephele.event.task.AbstractEvent;
import eu.stratosphere.nephele.event.task.AbstractTaskEvent;
import eu.stratosphere.nephele.execution.Environment;
import eu.stratosphere.runtime.io.channels.InputChannel;
import eu.stratosphere.nephele.jobgraph.JobID;
/**
* In Nephele input gates are a specialization of general gates and connect input channels and record readers. As
* channels, input gates are always parameterized to a specific type of record which they can transport. In contrast to
* output gates input gates can be associated with a {@link eu.stratosphere.runtime.io.serialization.io.DistributionPattern} object which dictates the concrete
* wiring between two groups of vertices.
*
* @param <T> The type of record that can be transported through this gate.
*/
public class InputGate<T extends IOReadableWritable> extends Gate<T> implements BufferProvider, LocalBufferPoolOwner {
/**
* The log object used for debugging.
*/
private static final Log LOG = LogFactory.getLog(InputGate.class);
/**
* The array of input channels attached to this input gate.
*/
private InputChannel<T>[] channels;
/**
* Queue with indices of channels that store at least one available record.
*/
private final BlockingQueue<Integer> availableChannels = new LinkedBlockingQueue<Integer>();
/**
* The listener object to be notified when a channel has at least one record available.
*/
private final AtomicReference<RecordAvailabilityListener<T>> recordAvailabilityListener = new AtomicReference<RecordAvailabilityListener<T>>(null);
private AbstractTaskEvent currentEvent;
/**
* If the value of this variable is set to <code>true</code>, the input gate is closed.
*/
private boolean isClosed = false;
/**
* The channel to read from next.
*/
private int channelToReadFrom = -1;
private LocalBufferPool bufferPool;
/**
* Constructs a new runtime input gate.
*
* @param jobID
* the ID of the job this input gate belongs to
* @param gateID
* the ID of the gate
* @param index
* the index assigned to this input gate at the {@link Environment} object
*/
public InputGate(final JobID jobID, final GateID gateID, final int index) {
super(jobID, gateID, index);
}
public void initializeChannels(GateDeploymentDescriptor inputGateDescriptor){
channels = new InputChannel[inputGateDescriptor.getNumberOfChannelDescriptors()];
setChannelType(inputGateDescriptor.getChannelType());
final int nicdd = inputGateDescriptor.getNumberOfChannelDescriptors();
for(int i = 0; i < nicdd; i++){
final ChannelDeploymentDescriptor cdd = inputGateDescriptor.getChannelDescriptor(i);
channels[i] = new InputChannel<T>(this, i, cdd.getInputChannelID(),
cdd.getOutputChannelID(), getChannelType());
}
}
@Override
public boolean isInputGate() {
return true;
}
/**
* Returns the number of input channels associated with this input gate.
*
* @return the number of input channels associated with this input gate
*/
public int getNumberOfInputChannels() {
return this.channels.length;
}
/**
* Returns the input channel from position <code>pos</code> of the gate's internal channel list.
*
* @param pos
* the position to retrieve the channel from
* @return the channel from the given position or <code>null</code> if such position does not exist.
*/
public InputChannel<T> getInputChannel(int pos) {
return this.channels[pos];
}
public InputChannel<T>[] channels() {
return this.channels;
}
/**
* Reads a record from one of the associated input channels. Channels are read such that one buffer from a channel is
* consecutively consumed. The buffers in turn are consumed in the order in which they arrive.
* Note that this method is not guaranteed to return a record, because the currently available channel data may not always
* constitute an entire record, when events or partial records are part of the data.
*
* When called even though no data is available, this call will block until data is available, so this method should be called
* when waiting is desired (such as when synchronously consuming a single gate) or only when it is known that data is available
* (such as when reading a union of multiple input gates).
*
* @param target The record object into which to construct the complete record.
* @return The result indicating whether a complete record is available, a event is available, only incomplete data
* is available (NONE), or the gate is exhausted.
* @throws IOException Thrown when an error occurred in the network stack relating to this channel.
* @throws InterruptedException Thrown, when the thread working on this channel is interrupted.
*/
public InputChannelResult readRecord(T target) throws IOException, InterruptedException {
if (this.channelToReadFrom == -1) {
if (this.isClosed()) {
return InputChannelResult.END_OF_STREAM;
}
if (Thread.interrupted()) {
throw new InterruptedException();
}
this.channelToReadFrom = waitForAnyChannelToBecomeAvailable();
}
InputChannelResult result = this.getInputChannel(this.channelToReadFrom).readRecord(target);
switch (result) {
case INTERMEDIATE_RECORD_FROM_BUFFER: // full record and we can stay on the same channel
return InputChannelResult.INTERMEDIATE_RECORD_FROM_BUFFER;
case LAST_RECORD_FROM_BUFFER: // full record, but we must switch the channel afterwards
this.channelToReadFrom = -1;
return InputChannelResult.LAST_RECORD_FROM_BUFFER;
case END_OF_SUPERSTEP:
this.channelToReadFrom = -1;
return InputChannelResult.END_OF_SUPERSTEP;
case TASK_EVENT: // task event
this.currentEvent = this.getInputChannel(this.channelToReadFrom).getCurrentEvent();
this.channelToReadFrom = -1; // event always marks a unit as consumed
return InputChannelResult.TASK_EVENT;
case NONE: // internal event or an incomplete record that needs further chunks
// the current unit is exhausted
this.channelToReadFrom = -1;
return InputChannelResult.NONE;
case END_OF_STREAM: // channel is done
this.channelToReadFrom = -1;
return isClosed() ? InputChannelResult.END_OF_STREAM : InputChannelResult.NONE;
default: // silence the compiler
throw new RuntimeException();
}
}
public AbstractTaskEvent getCurrentEvent() {
AbstractTaskEvent e = this.currentEvent;
this.currentEvent = null;
return e;
}
/**
* Notify the gate that the channel with the given index has
* at least one record available.
*
* @param channelIndex
* the index of the channel which has at least one record available
*/
public void notifyRecordIsAvailable(int channelIndex) {
this.availableChannels.add(Integer.valueOf(channelIndex));
RecordAvailabilityListener<T> listener = this.recordAvailabilityListener.get();
if (listener != null) {
listener.reportRecordAvailability(this);
}
}
/**
* This method returns the index of a channel which has at least
* one record available. The method may block until at least one
* channel has become ready.
*
* @return the index of the channel which has at least one record available
*/
public int waitForAnyChannelToBecomeAvailable() throws InterruptedException {
return this.availableChannels.take().intValue();
}
@Override
public boolean isClosed() throws IOException, InterruptedException {
if (this.isClosed) {
return true;
}
for (int i = 0; i < this.getNumberOfInputChannels(); i++) {
final InputChannel<T> inputChannel = this.channels[i];
if (!inputChannel.isClosed()) {
return false;
}
}
this.isClosed = true;
return true;
}
/**
* Immediately closes the input gate and all its input channels. The corresponding
* output channels are notified. Any remaining records in any buffers or queue is considered
* irrelevant and is discarded.
*
* @throws IOException
* thrown if an I/O error occurs while closing the gate
* @throws InterruptedException
* thrown if the thread is interrupted while waiting for the gate to be closed
*/
public void close() throws IOException, InterruptedException {
for (int i = 0; i < this.getNumberOfInputChannels(); i++) {
final InputChannel<T> inputChannel = this.channels[i];
inputChannel.close();
}
}
@Override
public String toString() {
return "Input " + super.toString();
}
@Override
public void publishEvent(AbstractEvent event) throws IOException, InterruptedException {
// Copy event to all connected channels
for(int i=0; i< getNumberOfChannels(); i++){
channels[i].transferEvent(event);
}
}
@Override
public void releaseAllChannelResources() {
for(int i=0; i< getNumberOfChannels(); i++){
channels[i].releaseAllResources();
}
}
/**
* Registers a {@link RecordAvailabilityListener} with this input gate.
*
* @param listener
* the listener object to be registered
*/
public void registerRecordAvailabilityListener(final RecordAvailabilityListener<T> listener) {
if (!this.recordAvailabilityListener.compareAndSet(null, listener)) {
throw new IllegalStateException(this.recordAvailabilityListener
+ " is already registered as a record availability listener");
}
}
/**
* Notify the gate that is has consumed a data unit from the channel with the given index
*
* @param channelIndex
* the index of the channel from which a data unit has been consumed
*/
public void notifyDataUnitConsumed(int channelIndex) {
this.channelToReadFrom = -1;
}
//
@Override
public Buffer requestBuffer(int minBufferSize) throws IOException {
return this.bufferPool.requestBuffer(minBufferSize);
}
@Override
public Buffer requestBufferBlocking(int minBufferSize) throws IOException, InterruptedException {
return this.bufferPool.requestBufferBlocking(minBufferSize);
}
@Override
public int getBufferSize() {
return this.bufferPool.getBufferSize();
}
@Override
public int getNumberOfChannels() {
return getNumberOfInputChannels();
}
@Override
public void setDesignatedNumberOfBuffers(int numBuffers) {
this.bufferPool.setNumDesignatedBuffers(numBuffers);
}
@Override
public void clearLocalBufferPool() {
this.bufferPool.destroy();
}
@Override
public void registerGlobalBufferPool(GlobalBufferPool globalBufferPool) {
this.bufferPool = new LocalBufferPool(globalBufferPool, 1);
}
@Override
public void logBufferUtilization() {
LOG.info(String.format("\t%s: %d available, %d requested, %d designated",
this,
this.bufferPool.numAvailableBuffers(),
this.bufferPool.numRequestedBuffers(),
this.bufferPool.numDesignatedBuffers()));
}
@Override
public void reportAsynchronousEvent() {
this.bufferPool.reportAsynchronousEvent();
}
@Override
public BufferAvailabilityRegistration registerBufferAvailabilityListener(BufferAvailabilityListener listener) {
return this.bufferPool.registerBufferAvailabilityListener(listener);
}
}