/***********************************************************************************************************************
* Copyright (C) 2010-2014 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.pact.runtime.task;
import eu.stratosphere.api.common.accumulators.Accumulator;
import eu.stratosphere.api.common.accumulators.AccumulatorHelper;
import eu.stratosphere.api.common.distributions.DataDistribution;
import eu.stratosphere.api.common.functions.Function;
import eu.stratosphere.api.common.functions.GenericCombine;
import eu.stratosphere.api.common.typeutils.TypeComparator;
import eu.stratosphere.api.common.typeutils.TypeComparatorFactory;
import eu.stratosphere.api.common.typeutils.TypeSerializer;
import eu.stratosphere.api.common.typeutils.TypeSerializerFactory;
import eu.stratosphere.configuration.Configuration;
import eu.stratosphere.core.io.IOReadableWritable;
import eu.stratosphere.nephele.execution.CancelTaskException;
import eu.stratosphere.nephele.execution.Environment;
import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager;
import eu.stratosphere.runtime.io.api.ChannelSelector;
import eu.stratosphere.runtime.io.api.RecordWriter;
import eu.stratosphere.runtime.io.api.MutableReader;
import eu.stratosphere.runtime.io.api.MutableRecordReader;
import eu.stratosphere.runtime.io.api.MutableUnionRecordReader;
import eu.stratosphere.runtime.io.api.BufferWriter;
import eu.stratosphere.nephele.services.accumulators.AccumulatorEvent;
import eu.stratosphere.nephele.services.iomanager.IOManager;
import eu.stratosphere.nephele.services.memorymanager.MemoryManager;
import eu.stratosphere.nephele.template.AbstractInputTask;
import eu.stratosphere.nephele.template.AbstractInvokable;
import eu.stratosphere.nephele.template.AbstractTask;
import eu.stratosphere.pact.runtime.plugable.DeserializationDelegate;
import eu.stratosphere.pact.runtime.plugable.SerializationDelegate;
import eu.stratosphere.pact.runtime.resettable.SpillingResettableMutableObjectIterator;
import eu.stratosphere.pact.runtime.shipping.OutputCollector;
import eu.stratosphere.pact.runtime.shipping.OutputEmitter;
import eu.stratosphere.pact.runtime.shipping.RecordOutputCollector;
import eu.stratosphere.pact.runtime.shipping.RecordOutputEmitter;
import eu.stratosphere.pact.runtime.shipping.ShipStrategyType;
import eu.stratosphere.pact.runtime.sort.CombiningUnilateralSortMerger;
import eu.stratosphere.pact.runtime.sort.UnilateralSortMerger;
import eu.stratosphere.pact.runtime.task.chaining.ChainedDriver;
import eu.stratosphere.pact.runtime.task.chaining.ExceptionInChainedStubException;
import eu.stratosphere.pact.runtime.task.util.CloseableInputProvider;
import eu.stratosphere.pact.runtime.task.util.LocalStrategy;
import eu.stratosphere.pact.runtime.task.util.ReaderIterator;
import eu.stratosphere.pact.runtime.task.util.RecordReaderIterator;
import eu.stratosphere.pact.runtime.task.util.TaskConfig;
import eu.stratosphere.pact.runtime.udf.RuntimeUDFContext;
import eu.stratosphere.types.Record;
import eu.stratosphere.util.Collector;
import eu.stratosphere.util.InstantiationUtil;
import eu.stratosphere.util.MutableObjectIterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* The abstract base class for all tasks. Encapsulated common behavior and implements the main life-cycle
* of the user code.
*/
public class RegularPactTask<S extends Function, OT> extends AbstractTask implements PactTaskContext<S, OT> {
protected static final Log LOG = LogFactory.getLog(RegularPactTask.class);
// --------------------------------------------------------------------------------------------
/**
* The driver that invokes the user code (the stub implementation). The central driver in this task
* (further drivers may be chained behind this driver).
*/
protected volatile PactDriver<S, OT> driver;
/**
* The instantiated user code of this task's main operator (driver). May be null if the operator has no udf.
*/
protected S stub;
/**
* The udf's runtime context.
*/
protected RuntimeUDFContext runtimeUdfContext;
/**
* The collector that forwards the user code's results. May forward to a channel or to chained drivers within
* this task.
*/
protected Collector<OT> output;
/**
* The output writers for the data that this task forwards to the next task. The latest driver (the central, if no chained
* drivers exist, otherwise the last chained driver) produces its output to these writers.
*/
protected List<BufferWriter> eventualOutputs;
/**
* The input readers to this task.
*/
protected MutableReader<?>[] inputReaders;
/**
* The input readers for the configured broadcast variables for this task.
*/
protected MutableReader<?>[] broadcastInputReaders;
/**
* The inputs reader, wrapped in an iterator. Prior to the local strategies, etc...
*/
protected MutableObjectIterator<?>[] inputIterators;
/**
* The input readers for the configured broadcast variables, wrapped in an iterator.
* Prior to the local strategies, etc...
*/
protected MutableObjectIterator<?>[] broadcastInputIterators;
protected int[] iterativeInputs;
protected int[] iterativeBroadcastInputs;
/**
* The local strategies that are applied on the inputs.
*/
protected volatile CloseableInputProvider<?>[] localStrategies;
/**
* The optional temp barriers on the inputs for dead-lock breaking. Are
* optionally resettable.
*/
protected volatile TempBarrier<?>[] tempBarriers;
/**
* The resettable inputs in the case where no temp barrier is needed.
*/
protected volatile SpillingResettableMutableObjectIterator<?>[] resettableInputs;
/**
* The inputs to the operator. Return the readers' data after the application of the local strategy
* and the temp-table barrier.
*/
protected MutableObjectIterator<?>[] inputs;
/**
* The serializers for the input data type.
*/
protected TypeSerializerFactory<?>[] inputSerializers;
/**
* The serializers for the broadcast input data types.
*/
protected TypeSerializerFactory<?>[] broadcastInputSerializers;
/**
* The comparators for the central driver.
*/
protected TypeComparator<?>[] inputComparators;
/**
* The task configuration with the setup parameters.
*/
protected TaskConfig config;
/**
* The class loader used to instantiate user code and user data types.
*/
protected ClassLoader userCodeClassLoader;
/**
* A list of chained drivers, if there are any.
*/
protected ArrayList<ChainedDriver<?, ?>> chainedTasks;
/**
* Certain inputs may be excluded from resetting. For example, the initial partial solution
* in an iteration head must not be reseted (it is read through the back channel), when all
* others are reseted.
*/
private boolean[] excludeFromReset;
/**
* Flag indicating for each input whether it is cached and can be reseted.
*/
private boolean[] inputIsCached;
/**
* flag indicating for each input whether it must be asynchronously materialized.
*/
private boolean[] inputIsAsyncMaterialized;
/**
* The amount of memory per input that is dedicated to the materialization.
*/
private int[] materializationMemory;
/**
* The flag that tags the task as still running. Checked periodically to abort processing.
*/
protected volatile boolean running = true;
// --------------------------------------------------------------------------------------------
// Task Interface
// --------------------------------------------------------------------------------------------
/**
* Initialization method. Runs in the execution graph setup phase in the JobManager
* and as a setup method on the TaskManager.
*/
@Override
public void registerInputOutput() {
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Start registering input and output."));
}
// get the classloader first. the classloader might have been set before by mock environments during testing
if (this.userCodeClassLoader == null) {
try {
this.userCodeClassLoader = LibraryCacheManager.getClassLoader(getEnvironment().getJobID());
}
catch (IOException ioe) {
throw new RuntimeException("The ClassLoader for the user code could not be instantiated from the library cache.", ioe);
}
}
// obtain task configuration (including stub parameters)
Configuration taskConf = getTaskConfiguration();
taskConf.setClassLoader(this.userCodeClassLoader);
this.config = new TaskConfig(taskConf);
// now get the operator class which drives the operation
final Class<? extends PactDriver<S, OT>> driverClass = this.config.getDriver();
this.driver = InstantiationUtil.instantiate(driverClass, PactDriver.class);
// initialize the readers. this is necessary for nephele to create the input gates
// however, this does not trigger any local processing.
try {
initInputReaders();
initBroadcastInputReaders();
} catch (Exception e) {
throw new RuntimeException("Initializing the input streams failed" +
e.getMessage() == null ? "." : ": " + e.getMessage(), e);
}
// initialize the writers. this is necessary for nephele to create the output gates.
// because in the presence of chained tasks, the tasks writers depend on the last task in the chain,
// we need to initialize the chained tasks as well. the chained tasks are only set up, but no work
// (such as setting up a sorter, etc.) starts
try {
initOutputs();
} catch (Exception e) {
throw new RuntimeException("Initializing the output handlers failed" +
e.getMessage() == null ? "." : ": " + e.getMessage(), e);
}
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Finished registering input and output."));
}
}
/**
* The main work method.
*/
@Override
public void invoke() throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Start task code."));
}
// whatever happens in this scope, make sure that the local strategies are cleaned up!
// note that the initialization of the local strategies is in the try-finally block as well,
// so that the thread that creates them catches its own errors that may happen in that process.
// this is especially important, since there may be asynchronous closes (such as through canceling).
try {
// initialize the serializers (one per channel) of the record writers
initOutputWriters(this.eventualOutputs);
// initialize the remaining data structures on the input and trigger the local processing
// the local processing includes building the dams / caches
try {
int numInputs = driver.getNumberOfInputs();
int numBroadcastInputs = this.config.getNumBroadcastInputs();
initInputsSerializersAndComparators(numInputs);
initBroadcastInputsSerializers(numBroadcastInputs);
// set the iterative status for inputs and broadcast inputs
{
List<Integer> iterativeInputs = new ArrayList<Integer>();
for (int i = 0; i < numInputs; i++) {
final int numberOfEventsUntilInterrupt = getTaskConfig().getNumberOfEventsUntilInterruptInIterativeGate(i);
if (numberOfEventsUntilInterrupt < 0) {
throw new IllegalArgumentException();
}
else if (numberOfEventsUntilInterrupt > 0) {
this.inputReaders[i].setIterative(numberOfEventsUntilInterrupt);
iterativeInputs.add(i);
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Input [" + i + "] reads in supersteps with [" +
+ numberOfEventsUntilInterrupt + "] event(s) till next superstep."));
}
}
}
this.iterativeInputs = asArray(iterativeInputs);
}
{
List<Integer> iterativeBcInputs = new ArrayList<Integer>();
for (int i = 0; i < numBroadcastInputs; i++) {
final int numberOfEventsUntilInterrupt = getTaskConfig().getNumberOfEventsUntilInterruptInIterativeBroadcastGate(i);
if (numberOfEventsUntilInterrupt < 0) {
throw new IllegalArgumentException();
}
else if (numberOfEventsUntilInterrupt > 0) {
this.broadcastInputReaders[i].setIterative(numberOfEventsUntilInterrupt);
iterativeBcInputs.add(i);
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Broadcast input [" + i + "] reads in supersteps with [" +
+ numberOfEventsUntilInterrupt + "] event(s) till next superstep."));
}
}
}
this.iterativeBroadcastInputs = asArray(iterativeBcInputs);
}
initLocalStrategies(numInputs);
} catch (Exception e) {
throw new RuntimeException("Initializing the input processing failed" +
e.getMessage() == null ? "." : ": " + e.getMessage(), e);
}
if (!this.running) {
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Task cancelled before task code was started."));
}
return;
}
// pre main-function initialization
initialize();
// read the broadcast variables
for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
final String name = this.config.getBroadcastInputName(i);
readAndSetBroadcastInput(i, name, this.runtimeUdfContext);
}
// the work goes here
run();
}
finally {
// clean up in any case!
closeLocalStrategiesAndCaches();
}
if (this.running) {
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Finished task code."));
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Task code cancelled."));
}
}
}
@Override
public void cancel() throws Exception {
this.running = false;
if (LOG.isDebugEnabled()) {
LOG.debug(formatLogString("Cancelling task code"));
}
try {
if (this.driver != null) {
this.driver.cancel();
}
} finally {
closeLocalStrategiesAndCaches();
}
}
/**
* Sets the class-loader to be used to load the user code.
*
* @param cl The class-loader to be used to load the user code.
*/
public void setUserCodeClassLoader(ClassLoader cl) {
this.userCodeClassLoader = cl;
}
// --------------------------------------------------------------------------------------------
// Main Work Methods
// --------------------------------------------------------------------------------------------
protected void initialize() throws Exception {
// create the operator
try {
this.driver.setup(this);
}
catch (Throwable t) {
throw new Exception("The driver setup for '" + this.getEnvironment().getTaskName() +
"' , caused an error: " + t.getMessage(), t);
}
this.runtimeUdfContext = createRuntimeContext(getEnvironment().getTaskName());
// instantiate the UDF
try {
final Class<? super S> userCodeFunctionType = this.driver.getStubType();
// if the class is null, the driver has no user code
if (userCodeFunctionType != null) {
this.stub = initStub(userCodeFunctionType);
}
} catch (Exception e) {
throw new RuntimeException("Initializing the UDF" +
e.getMessage() == null ? "." : ": " + e.getMessage(), e);
}
}
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, RuntimeUDFContext context) throws IOException {
// drain the broadcast inputs
@SuppressWarnings("unchecked")
final MutableObjectIterator<X> reader = (MutableObjectIterator<X>) this.broadcastInputIterators[inputNum];
@SuppressWarnings("unchecked")
final TypeSerializer<X> serializer = (TypeSerializer<X>) this.broadcastInputSerializers[inputNum].getSerializer();
ArrayList<X> collection = new ArrayList<X>();
X record = serializer.createInstance();
while (this.running && ((record = reader.next(record)) != null)) {
collection.add(record);
record = serializer.createInstance();
}
context.setBroadcastVariable(bcVarName, collection);
}
protected void run() throws Exception {
// ---------------------------- Now, the actual processing starts ------------------------
// check for asynchronous canceling
if (!this.running) {
return;
}
boolean stubOpen = false;
try {
// run the data preparation
try {
this.driver.prepare();
}
catch (Throwable t) {
// if the preparation caused an error, clean up
// errors during clean-up are swallowed, because we have already a root exception
throw new Exception("The data preparation for task '" + this.getEnvironment().getTaskName() +
"' , caused an error: " + t.getMessage(), t);
}
// check for canceling
if (!this.running) {
return;
}
// start all chained tasks
RegularPactTask.openChainedTasks(this.chainedTasks, this);
// open stub implementation
if (this.stub != null) {
try {
Configuration stubConfig = this.config.getStubParameters();
this.stub.open(stubConfig);
stubOpen = true;
}
catch (Throwable t) {
throw new Exception("The user defined 'open()' method caused an exception: " + t.getMessage(), t);
}
}
// run the user code
this.driver.run();
// close. We close here such that a regular close throwing an exception marks a task as failed.
if (this.running && this.stub != null) {
this.stub.close();
stubOpen = false;
}
this.output.close();
// close all chained tasks letting them report failure
RegularPactTask.closeChainedTasks(this.chainedTasks, this);
// Collect the accumulators of all involved UDFs and send them to the
// JobManager. close() has been called earlier for all involved UDFs
// (using this.stub.close() and closeChainedTasks()), so UDFs can no longer
// modify accumulators.ll;
if (this.stub != null) {
// collect the counters from the stub
Map<String, Accumulator<?,?>> accumulators = this.stub.getRuntimeContext().getAllAccumulators();
RegularPactTask.reportAndClearAccumulators(getEnvironment(), accumulators, this.chainedTasks);
}
}
catch (Exception ex) {
// close the input, but do not report any exceptions, since we already have another root cause
if (stubOpen) {
try {
this.stub.close();
}
catch (Throwable t) {}
}
RegularPactTask.cancelChainedTasks(this.chainedTasks);
ex = ExceptionInChainedStubException.exceptionUnwrap(ex);
if (ex instanceof CancelTaskException) {
// forward canceling exception
throw ex;
}
else if (this.running) {
// throw only if task was not cancelled. in the case of canceling, exceptions are expected
RegularPactTask.logAndThrowException(ex, this);
}
}
finally {
this.driver.cleanup();
}
}
/**
* This method is called at the end of a task, receiving the accumulators of
* the task and the chained tasks. It merges them into a single map of
* accumulators and sends them to the JobManager.
*
* @param chainedTasks
* Each chained task might have accumulators which will be merged
* with the accumulators of the stub.
*/
protected static void reportAndClearAccumulators(Environment env, Map<String, Accumulator<?, ?>> accumulators,
ArrayList<ChainedDriver<?, ?>> chainedTasks) {
// We can merge here the accumulators from the stub and the chained
// tasks. Type conflicts can occur here if counters with same name but
// different type were used.
for (ChainedDriver<?, ?> chainedTask : chainedTasks) {
Map<String, Accumulator<?, ?>> chainedAccumulators = chainedTask.getStub().getRuntimeContext().getAllAccumulators();
AccumulatorHelper.mergeInto(accumulators, chainedAccumulators);
}
// Don't report if the UDF didn't collect any accumulators
if (accumulators.size() == 0) {
return;
}
// Report accumulators to JobManager
synchronized (env.getAccumulatorProtocolProxy()) {
try {
env.getAccumulatorProtocolProxy().reportAccumulatorResult(
new AccumulatorEvent(env.getJobID(), accumulators, true));
} catch (IOException e) {
throw new RuntimeException("Communication with JobManager is broken. Could not send accumulators.", e);
}
}
// We also clear the accumulators, since stub instances might be reused
// (e.g. in iterations) and we don't want to count twice. This may not be
// done before sending
AccumulatorHelper.resetAndClearAccumulators(accumulators);
for (ChainedDriver<?, ?> chainedTask : chainedTasks) {
AccumulatorHelper.resetAndClearAccumulators(chainedTask.getStub().getRuntimeContext().getAllAccumulators());
}
}
protected void closeLocalStrategiesAndCaches() {
if (this.localStrategies != null) {
for (int i = 0; i < this.localStrategies.length; i++) {
if (this.localStrategies[i] != null) {
try {
this.localStrategies[i].close();
} catch (Throwable t) {
LOG.error("Error closing local strategy for input " + i, t);
}
}
}
}
if (this.tempBarriers != null) {
for (int i = 0; i < this.tempBarriers.length; i++) {
if (this.tempBarriers[i] != null) {
try {
this.tempBarriers[i].close();
} catch (Throwable t) {
LOG.error("Error closing temp barrier for input " + i, t);
}
}
}
}
if (this.resettableInputs != null) {
for (int i = 0; i < this.resettableInputs.length; i++) {
if (this.resettableInputs[i] != null) {
try {
this.resettableInputs[i].close();
} catch (Throwable t) {
LOG.error("Error closing cache for input " + i, t);
}
}
}
}
}
// --------------------------------------------------------------------------------------------
// Task Setup and Teardown
// --------------------------------------------------------------------------------------------
/**
* @return the last output collector in the collector chain
*/
@SuppressWarnings("unchecked")
protected Collector<OT> getLastOutputCollector() {
int numChained = this.chainedTasks.size();
return (numChained == 0) ? output : (Collector<OT>) chainedTasks.get(numChained - 1).getOutputCollector();
}
/**
* Sets the last output {@link Collector} of the collector chain of this {@link RegularPactTask}.
* <p/>
* In case of chained tasks, the output collector of the last {@link ChainedDriver} is set. Otherwise it is the
* single collector of the {@link RegularPactTask}.
*
* @param newOutputCollector new output collector to set as last collector
*/
protected void setLastOutputCollector(Collector<OT> newOutputCollector) {
int numChained = this.chainedTasks.size();
if (numChained == 0) {
output = newOutputCollector;
return;
}
chainedTasks.get(numChained - 1).setOutputCollector(newOutputCollector);
}
public TaskConfig getLastTasksConfig() {
int numChained = this.chainedTasks.size();
return (numChained == 0) ? config : chainedTasks.get(numChained - 1).getTaskConfig();
}
protected S initStub(Class<? super S> stubSuperClass) throws Exception {
try {
S stub = config.<S>getStubWrapper(this.userCodeClassLoader).getUserCodeObject(stubSuperClass, this.userCodeClassLoader);
// check if the class is a subclass, if the check is required
if (stubSuperClass != null && !stubSuperClass.isAssignableFrom(stub.getClass())) {
Thread.dumpStack();
throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" +
stubSuperClass.getName() + "' as is required.");
}
stub.setRuntimeContext(this.runtimeUdfContext);
return stub;
}
catch (ClassCastException ccex) {
throw new Exception("The stub class is not a proper subclass of " + stubSuperClass.getName(), ccex);
}
}
/**
* Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
*
* This method requires that the task configuration, the driver, and the user-code class loader are set.
*/
@SuppressWarnings("unchecked")
protected void initInputReaders() throws Exception {
final int numInputs = getNumTaskInputs();
final MutableReader<?>[] inputReaders = new MutableReader[numInputs];
int numGates = 0;
for (int i = 0; i < numInputs; i++) {
// ---------------- create the input readers ---------------------
// in case where a logical input unions multiple physical inputs, create a union reader
final int groupSize = this.config.getGroupSize(i);
numGates += groupSize;
if (groupSize == 1) {
// non-union case
inputReaders[i] = new MutableRecordReader<IOReadableWritable>(this);
} else if (groupSize > 1){
// union case
MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
for (int j = 0; j < groupSize; ++j) {
readers[j] = new MutableRecordReader<IOReadableWritable>(this);
}
inputReaders[i] = new MutableUnionRecordReader<IOReadableWritable>(readers);
} else {
throw new Exception("Illegal input group size in task configuration: " + groupSize);
}
}
this.inputReaders = inputReaders;
// final sanity check
if (numGates != this.config.getNumInputs()) {
throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
}
}
/**
* Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
*
* This method requires that the task configuration, the driver, and the user-code class loader are set.
*/
@SuppressWarnings("unchecked")
protected void initBroadcastInputReaders() throws Exception {
final int numBroadcastInputs = this.config.getNumBroadcastInputs();
final MutableReader<?>[] broadcastInputReaders = new MutableReader[numBroadcastInputs];
for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
// ---------------- create the input readers ---------------------
// in case where a logical input unions multiple physical inputs, create a union reader
final int groupSize = this.config.getBroadcastGroupSize(i);
if (groupSize == 1) {
// non-union case
broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(this);
} else if (groupSize > 1){
// union case
MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
for (int j = 0; j < groupSize; ++j) {
readers[j] = new MutableRecordReader<IOReadableWritable>(this);
}
broadcastInputReaders[i] = new MutableUnionRecordReader<IOReadableWritable>(readers);
} else {
throw new Exception("Illegal input group size in task configuration: " + groupSize);
}
}
this.broadcastInputReaders = broadcastInputReaders;
}
/**
* Creates all the serializers and comparators.
*/
protected void initInputsSerializersAndComparators(int numInputs) throws Exception {
this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
this.inputComparators = this.driver.requiresComparatorOnInput() ? new TypeComparator[numInputs] : null;
this.inputIterators = new MutableObjectIterator[numInputs];
for (int i = 0; i < numInputs; i++) {
// ---------------- create the serializer first ---------------------
final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, this.userCodeClassLoader);
this.inputSerializers[i] = serializerFactory;
// ---------------- create the driver's comparator ---------------------
if (this.inputComparators != null) {
final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, this.userCodeClassLoader);
this.inputComparators[i] = comparatorFactory.createComparator();
}
this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
}
}
/**
* Creates all the serializers and iterators for the broadcast inputs.
*/
protected void initBroadcastInputsSerializers(int numBroadcastInputs) throws Exception {
this.broadcastInputSerializers = new TypeSerializerFactory[numBroadcastInputs];
this.broadcastInputIterators = new MutableObjectIterator[numBroadcastInputs];
for (int i = 0; i < numBroadcastInputs; i++) {
// ---------------- create the serializer first ---------------------
final TypeSerializerFactory<?> serializerFactory = this.config.getBroadcastInputSerializer(i, this.userCodeClassLoader);
this.broadcastInputSerializers[i] = serializerFactory;
this.broadcastInputIterators[i] = createInputIterator(this.broadcastInputReaders[i], this.broadcastInputSerializers[i]);
}
}
/**
*
* NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
* {@code #initInputSerializersAndComparators(int)}!
*
* @param numInputs
*/
protected void initLocalStrategies(int numInputs) throws Exception {
final MemoryManager memMan = getMemoryManager();
final IOManager ioMan = getIOManager();
this.localStrategies = new CloseableInputProvider[numInputs];
this.inputs = new MutableObjectIterator[numInputs];
this.excludeFromReset = new boolean[numInputs];
this.inputIsCached = new boolean[numInputs];
this.inputIsAsyncMaterialized = new boolean[numInputs];
this.materializationMemory = new int[numInputs];
// set up the local strategies first, such that the can work before any temp barrier is created
for (int i = 0; i < numInputs; i++) {
initInputLocalStrategy(i);
}
// we do another loop over the inputs, because we want to instantiate all
// sorters, etc before requesting the first input (as this call may block)
// we have two types of materialized inputs, and both are replayable (can act as a cache)
// The first variant materializes in a different thread and hence
// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
// the second variant spills to the side and will not read unless the result is also consumed
// in a pipelined fashion.
this.resettableInputs = new SpillingResettableMutableObjectIterator[numInputs];
this.tempBarriers = new TempBarrier[numInputs];
for (int i = 0; i < numInputs; i++) {
final int memoryPages;
final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
final boolean cached = this.config.isInputCached(i);
this.inputIsAsyncMaterialized[i] = async;
this.inputIsCached[i] = cached;
if (async || cached) {
memoryPages = memMan.computeNumberOfPages(this.config.getInputMaterializationMemory(i));
if (memoryPages <= 0) {
throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
}
this.materializationMemory[i] = memoryPages;
} else {
memoryPages = 0;
}
if (async) {
@SuppressWarnings({ "unchecked", "rawtypes" })
TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
barrier.startReading();
this.tempBarriers[i] = barrier;
this.inputs[i] = null;
} else if (cached) {
@SuppressWarnings({ "unchecked", "rawtypes" })
SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
this.resettableInputs[i] = iter;
this.inputs[i] = iter;
}
}
}
protected void resetAllInputs() throws Exception {
// close all local-strategies. they will either get re-initialized, or we have
// read them now and their data is cached
for (int i = 0; i < this.localStrategies.length; i++) {
if (this.localStrategies[i] != null) {
this.localStrategies[i].close();
this.localStrategies[i] = null;
}
}
final MemoryManager memMan = getMemoryManager();
final IOManager ioMan = getIOManager();
// reset the caches, or re-run the input local strategy
for (int i = 0; i < this.inputs.length; i++) {
if (this.excludeFromReset[i]) {
if (this.tempBarriers[i] != null) {
this.tempBarriers[i].close();
this.tempBarriers[i] = null;
} else if (this.resettableInputs[i] != null) {
this.resettableInputs[i].close();
this.resettableInputs[i] = null;
}
} else {
// make sure the input is not available directly, but are lazily fetched again
this.inputs[i] = null;
if (this.inputIsCached[i]) {
if (this.tempBarriers[i] != null) {
this.inputs[i] = this.tempBarriers[i].getIterator();
} else if (this.resettableInputs[i] != null) {
this.resettableInputs[i].consumeAndCacheRemainingData();
this.resettableInputs[i].reset();
this.inputs[i] = this.resettableInputs[i];
} else {
throw new RuntimeException("Found a resettable input, but no temp barrier and no resettable iterator.");
}
} else {
// close the async barrier if there is one
if (this.tempBarriers[i] != null) {
this.tempBarriers[i].close();
}
// recreate the local strategy
initInputLocalStrategy(i);
if (this.inputIsAsyncMaterialized[i]) {
final int pages = this.materializationMemory[i];
@SuppressWarnings({ "unchecked", "rawtypes" })
TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, pages);
barrier.startReading();
this.tempBarriers[i] = barrier;
this.inputs[i] = null;
}
}
}
}
}
protected void excludeFromReset(int inputNum) {
this.excludeFromReset[inputNum] = true;
}
private void initInputLocalStrategy(int inputNum) throws Exception {
// check if there is already a strategy
if (this.localStrategies[inputNum] != null) {
throw new IllegalStateException();
}
// now set up the local strategy
final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
if (localStrategy != null) {
switch (localStrategy) {
case NONE:
// the input is as it is
this.inputs[inputNum] = this.inputIterators[inputNum];
break;
case SORT:
@SuppressWarnings({ "rawtypes", "unchecked" })
UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
this.config.getMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
this.config.getSpillingThresholdInput(inputNum));
// set the input to null such that it will be lazily fetched from the input strategy
this.inputs[inputNum] = null;
this.localStrategies[inputNum] = sorter;
break;
case COMBININGSORT:
// sanity check this special case!
// this still breaks a bit of the abstraction!
// we should have nested configurations for the local strategies to solve that
if (inputNum != 0) {
throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
}
// instantiate ourselves a combiner. we should not use the stub, because the sort and the
// subsequent (group)reduce would otherwise share it multi-threaded
final Class<S> userCodeFunctionType = this.driver.getStubType();
if (userCodeFunctionType == null) {
throw new IllegalStateException("Performing combining sort outside a reduce task!");
}
final S localStub;
try {
localStub = initStub(userCodeFunctionType);
} catch (Exception e) {
throw new RuntimeException("Initializing the user code and the configuration failed" +
e.getMessage() == null ? "." : ": " + e.getMessage(), e);
}
if (!(localStub instanceof GenericCombine)) {
throw new IllegalStateException("Performing combining sort outside a reduce task!");
}
@SuppressWarnings({ "rawtypes", "unchecked" })
CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
(GenericCombine) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
this.config.getMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
this.config.getSpillingThresholdInput(inputNum));
cSorter.setUdfConfiguration(this.config.getStubParameters());
// set the input to null such that it will be lazily fetched from the input strategy
this.inputs[inputNum] = null;
this.localStrategies[inputNum] = cSorter;
break;
default:
throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
}
} else {
// no local strategy in the config
this.inputs[inputNum] = this.inputIterators[inputNum];
}
}
private <T> TypeComparator<T> getLocalStrategyComparator(int inputNum) throws Exception {
TypeComparatorFactory<T> compFact = this.config.getInputComparator(inputNum, this.userCodeClassLoader);
if (compFact == null) {
throw new Exception("Missing comparator factory for local strategy on input " + inputNum);
}
return compFact.createComparator();
}
protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
if (serializerFactory.getDataType().equals(Record.class)) {
// record specific deserialization
@SuppressWarnings("unchecked")
MutableReader<Record> reader = (MutableReader<Record>) inputReader;
return new RecordReaderIterator(reader);
} else {
// generic data type serialization
@SuppressWarnings("unchecked")
MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
@SuppressWarnings({ "unchecked", "rawtypes" })
final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
return iter;
}
// // generic data type serialization
// @SuppressWarnings("unchecked")
// MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
// @SuppressWarnings({ "unchecked", "rawtypes" })
// final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializer);
// return iter;
}
protected int getNumTaskInputs() {
return this.driver.getNumberOfInputs();
}
/**
* Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
* The output collector applies the configured shipping strategies for each writer.
*/
protected void initOutputs() throws Exception {
this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
this.eventualOutputs = new ArrayList<BufferWriter>();
this.output = initOutputs(this, this.userCodeClassLoader, this.config, this.chainedTasks, this.eventualOutputs);
}
public RuntimeUDFContext createRuntimeContext(String taskName) {
Environment env = getEnvironment();
return new RuntimeUDFContext(taskName, env.getCurrentNumberOfSubtasks(), env.getIndexInSubtaskGroup(), env.getCopyTask());
}
// --------------------------------------------------------------------------------------------
// Task Context Signature
// -------------------------------------------------------------------------------------------
@Override
public TaskConfig getTaskConfig() {
return this.config;
}
@Override
public ClassLoader getUserCodeClassLoader() {
return this.userCodeClassLoader;
}
@Override
public MemoryManager getMemoryManager() {
return getEnvironment().getMemoryManager();
}
@Override
public IOManager getIOManager() {
return getEnvironment().getIOManager();
}
@Override
public S getStub() {
return this.stub;
}
@Override
public Collector<OT> getOutputCollector() {
return this.output;
}
@Override
public AbstractInvokable getOwningNepheleTask() {
return this;
}
@Override
public String formatLogString(String message) {
return constructLogString(message, getEnvironment().getTaskName(), this);
}
@Override
public <X> MutableObjectIterator<X> getInput(int index) {
if (index < 0 || index > this.driver.getNumberOfInputs()) {
throw new IndexOutOfBoundsException();
}
// check for lazy assignment from input strategies
if (this.inputs[index] != null) {
@SuppressWarnings("unchecked")
MutableObjectIterator<X> in = (MutableObjectIterator<X>) this.inputs[index];
return in;
} else {
final MutableObjectIterator<X> in;
try {
if (this.tempBarriers[index] != null) {
@SuppressWarnings("unchecked")
MutableObjectIterator<X> iter = (MutableObjectIterator<X>) this.tempBarriers[index].getIterator();
in = iter;
} else if (this.localStrategies[index] != null) {
@SuppressWarnings("unchecked")
MutableObjectIterator<X> iter = (MutableObjectIterator<X>) this.localStrategies[index].getIterator();
in = iter;
} else {
throw new RuntimeException("Bug: null input iterator, null temp barrier, and null local strategy.");
}
this.inputs[index] = in;
return in;
} catch (InterruptedException iex) {
throw new RuntimeException("Interrupted while waiting for input " + index + " to become available.");
} catch (IOException ioex) {
throw new RuntimeException("An I/O Exception occurred whily obaining input " + index + ".");
}
}
}
@Override
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
if (index < 0 || index >= this.driver.getNumberOfInputs()) {
throw new IndexOutOfBoundsException();
}
@SuppressWarnings("unchecked")
final TypeSerializerFactory<X> serializerFactory = (TypeSerializerFactory<X>) this.inputSerializers[index];
return serializerFactory;
}
@Override
public <X> TypeComparator<X> getInputComparator(int index) {
if (this.inputComparators == null) {
throw new IllegalStateException("Comparators have not been created!");
}
else if (index < 0 || index >= this.driver.getNumberOfInputs()) {
throw new IndexOutOfBoundsException();
}
@SuppressWarnings("unchecked")
final TypeComparator<X> comparator = (TypeComparator<X>) this.inputComparators[index];
return comparator;
}
// ============================================================================================
// Static Utilities
//
// Utilities are consolidated here to ensure a uniform way of running,
// logging, exception handling, and error messages.
// ============================================================================================
// --------------------------------------------------------------------------------------------
// Logging
// --------------------------------------------------------------------------------------------
/**
* Utility function that composes a string for logging purposes. The string includes the given message,
* the given name of the task and the index in its subtask group as well as the number of instances
* that exist in its subtask group.
*
* @param message The main message for the log.
* @param taskName The name of the task.
* @param parent The nephele task that contains the code producing the message.
*
* @return The string for logging.
*/
public static String constructLogString(String message, String taskName, AbstractInvokable parent) {
return message + ": " + taskName + " (" + (parent.getEnvironment().getIndexInSubtaskGroup() + 1) +
'/' + parent.getEnvironment().getCurrentNumberOfSubtasks() + ')';
}
/**
* Prints an error message and throws the given exception. If the exception is of the type
* {@link ExceptionInChainedStubException} then the chain of contained exceptions is followed
* until an exception of a different type is found.
*
* @param ex The exception to be thrown.
* @param parent The parent task, whose information is included in the log message.
* @throws Exception Always thrown.
*/
public static void logAndThrowException(Exception ex, AbstractInvokable parent) throws Exception {
String taskName;
if (ex instanceof ExceptionInChainedStubException) {
do {
ExceptionInChainedStubException cex = (ExceptionInChainedStubException) ex;
taskName = cex.getTaskName();
ex = cex.getWrappedException();
} while (ex instanceof ExceptionInChainedStubException);
} else {
taskName = parent.getEnvironment().getTaskName();
}
if (LOG.isErrorEnabled()) {
LOG.error(constructLogString("Error in task code", taskName, parent), ex);
}
throw ex;
}
// --------------------------------------------------------------------------------------------
// Result Shipping and Chained Tasks
// --------------------------------------------------------------------------------------------
/**
* Creates the {@link Collector} for the given task, as described by the given configuration. The
* output collector contains the writers that forward the data to the different tasks that the given task
* is connected to. Each writer applies a the partitioning as described in the configuration.
*
* @param task The task that the output collector is created for.
* @param config The configuration describing the output shipping strategies.
* @param cl The classloader used to load user defined types.
* @param numOutputs The number of outputs described in the configuration.
*
* @return The OutputCollector that data produced in this task is submitted to.
*/
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl, List<BufferWriter> eventualOutputs, int numOutputs)
throws Exception
{
if (numOutputs <= 0) {
throw new Exception("BUG: The task must have at least one output");
}
// get the factory for the serializer
final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
// special case the Record
if (serializerFactory.getDataType().equals(Record.class)) {
final List<RecordWriter<Record>> writers = new ArrayList<RecordWriter<Record>>(numOutputs);
// create a writer for each output
for (int i = 0; i < numOutputs; i++) {
// create the OutputEmitter from output ship strategy
final ShipStrategyType strategy = config.getOutputShipStrategy(i);
final TypeComparatorFactory<?> compFact = config.getOutputComparator(i, cl);
final RecordOutputEmitter oe;
if (compFact == null) {
oe = new RecordOutputEmitter(strategy);
} else {
@SuppressWarnings("unchecked")
TypeComparator<Record> comparator = (TypeComparator<Record>) compFact.createComparator();
if (!comparator.supportsCompareAgainstReference()) {
throw new Exception("Incompatibe serializer-/comparator factories.");
}
final DataDistribution distribution = config.getOutputDataDistribution(i, cl);
oe = new RecordOutputEmitter(strategy, comparator, distribution);
}
if (task instanceof AbstractTask) {
writers.add(new RecordWriter<Record>((AbstractTask) task, oe));
} else if (task instanceof AbstractInputTask<?>) {
writers.add(new RecordWriter<Record>((AbstractInputTask<?>) task, oe));
}
}
if (eventualOutputs != null) {
eventualOutputs.addAll(writers);
}
@SuppressWarnings("unchecked")
final Collector<T> outColl = (Collector<T>) new RecordOutputCollector(writers);
return outColl;
}
else {
// generic case
final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<RecordWriter<SerializationDelegate<T>>>(numOutputs);
// create a writer for each output
for (int i = 0; i < numOutputs; i++)
{
// create the OutputEmitter from output ship strategy
final ShipStrategyType strategy = config.getOutputShipStrategy(i);
final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);
final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
final ChannelSelector<SerializationDelegate<T>> oe;
if (compFactory == null) {
oe = new OutputEmitter<T>(strategy);
} else if (dataDist == null){
final TypeComparator<T> comparator = compFactory.createComparator();
oe = new OutputEmitter<T>(strategy, comparator);
} else {
final TypeComparator<T> comparator = compFactory.createComparator();
oe = new OutputEmitter<T>(strategy, comparator, dataDist);
}
if (task instanceof AbstractTask) {
writers.add(new RecordWriter<SerializationDelegate<T>>((AbstractTask) task, oe));
} else if (task instanceof AbstractInputTask<?>) {
writers.add(new RecordWriter<SerializationDelegate<T>>((AbstractInputTask<?>) task, oe));
}
}
if (eventualOutputs != null) {
eventualOutputs.addAll(writers);
}
return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}
}
/**
* Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
* The output collector applies the configured shipping strategy.
*/
@SuppressWarnings("unchecked")
public static <T> Collector<T> initOutputs(AbstractInvokable nepheleTask, ClassLoader cl, TaskConfig config,
List<ChainedDriver<?, ?>> chainedTasksTarget, List<BufferWriter> eventualOutputs)
throws Exception
{
final int numOutputs = config.getNumOutputs();
// check whether we got any chained tasks
final int numChained = config.getNumberOfChainedStubs();
if (numChained > 0) {
// got chained stubs. that means that this one may only have a single forward connection
if (numOutputs != 1 || config.getOutputShipStrategy(0) != ShipStrategyType.FORWARD) {
throw new RuntimeException("Plan Generation Bug: Found a chained stub that is not connected via an only forward connection.");
}
// instantiate each task
@SuppressWarnings("rawtypes")
Collector previous = null;
for (int i = numChained - 1; i >= 0; --i)
{
// get the task first
final ChainedDriver<?, ?> ct;
try {
Class<? extends ChainedDriver<?, ?>> ctc = config.getChainedTask(i);
ct = ctc.newInstance();
}
catch (Exception ex) {
throw new RuntimeException("Could not instantiate chained task driver.", ex);
}
// get the configuration for the task
final TaskConfig chainedStubConf = config.getChainedStubConfig(i);
final String taskName = config.getChainedTaskName(i);
if (i == numChained -1) {
// last in chain, instantiate the output collector for this task
previous = getOutputCollector(nepheleTask, chainedStubConf, cl, eventualOutputs, chainedStubConf.getNumOutputs());
}
ct.setup(chainedStubConf, taskName, previous, nepheleTask, cl);
chainedTasksTarget.add(0, ct);
previous = ct;
}
// the collector of the first in the chain is the collector for the nephele task
return (Collector<T>) previous;
}
// else
// instantiate the output collector the default way from this configuration
return getOutputCollector(nepheleTask , config, cl, eventualOutputs, numOutputs);
}
public static void initOutputWriters(List<BufferWriter> writers) {
for (BufferWriter writer : writers) {
((RecordWriter<?>) writer).initializeSerializers();
}
}
// --------------------------------------------------------------------------------------------
// User Code LifeCycle
// --------------------------------------------------------------------------------------------
/**
* Opens the given stub using its {@link Function#open(Configuration)} method. If the open call produces
* an exception, a new exception with a standard error message is created, using the encountered exception
* as its cause.
*
* @param stub The user code instance to be opened.
* @param parameters The parameters supplied to the user code.
*
* @throws Exception Thrown, if the user code's open method produces an exception.
*/
public static void openUserCode(Function stub, Configuration parameters) throws Exception {
try {
stub.open(parameters);
} catch (Throwable t) {
throw new Exception("The user defined 'open(Configuration)' method in " + stub.getClass().toString() + " caused an exception: " + t.getMessage(), t);
}
}
/**
* Closes the given stub using its {@link Function#close()} method. If the close call produces
* an exception, a new exception with a standard error message is created, using the encountered exception
* as its cause.
*
* @param stub The user code instance to be closed.
*
* @throws Exception Thrown, if the user code's close method produces an exception.
*/
public static void closeUserCode(Function stub) throws Exception {
try {
stub.close();
} catch (Throwable t) {
throw new Exception("The user defined 'close()' method caused an exception: " + t.getMessage(), t);
}
}
// --------------------------------------------------------------------------------------------
// Chained Task LifeCycle
// --------------------------------------------------------------------------------------------
/**
* Opens all chained tasks, in the order as they are stored in the array. The opening process
* creates a standardized log info message.
*
* @param tasks The tasks to be opened.
* @param parent The parent task, used to obtain parameters to include in the log message.
* @throws Exception Thrown, if the opening encounters an exception.
*/
public static void openChainedTasks(List<ChainedDriver<?, ?>> tasks, AbstractInvokable parent) throws Exception {
// start all chained tasks
for (int i = 0; i < tasks.size(); i++) {
final ChainedDriver<?, ?> task = tasks.get(i);
if (LOG.isDebugEnabled()) {
LOG.debug(constructLogString("Start task code", task.getTaskName(), parent));
}
task.openTask();
}
}
/**
* Closes all chained tasks, in the order as they are stored in the array. The closing process
* creates a standardized log info message.
*
* @param tasks The tasks to be closed.
* @param parent The parent task, used to obtain parameters to include in the log message.
* @throws Exception Thrown, if the closing encounters an exception.
*/
public static void closeChainedTasks(List<ChainedDriver<?, ?>> tasks, AbstractInvokable parent) throws Exception {
for (int i = 0; i < tasks.size(); i++) {
final ChainedDriver<?, ?> task = tasks.get(i);
task.closeTask();
if (LOG.isDebugEnabled()) {
LOG.debug(constructLogString("Finished task code", task.getTaskName(), parent));
}
}
}
/**
* Cancels all tasks via their {@link ChainedDriver#cancelTask()} method. Any occurring exception
* and error is suppressed, such that the canceling method of every task is invoked in all cases.
*
* @param tasks The tasks to be canceled.
*/
public static void cancelChainedTasks(List<ChainedDriver<?, ?>> tasks) {
for (int i = 0; i < tasks.size(); i++) {
try {
tasks.get(i).cancelTask();
} catch (Throwable t) {}
}
}
// --------------------------------------------------------------------------------------------
// Miscellaneous Utilities
// --------------------------------------------------------------------------------------------
/**
* Instantiates a user code class from is definition in the task configuration.
* The class is instantiated without arguments using the null-ary constructor. Instantiation
* will fail if this constructor does not exist or is not public.
*
* @param <T> The generic type of the user code class.
* @param config The task configuration containing the class description.
* @param cl The class loader to be used to load the class.
* @param superClass The super class that the user code class extends or implements, for type checking.
*
* @return An instance of the user code class.
*/
public static <T> T instantiateUserCode(TaskConfig config, ClassLoader cl, Class<? super T> superClass) {
try {
T stub = config.<T>getStubWrapper(cl).getUserCodeObject(superClass, cl);
// check if the class is a subclass, if the check is required
if (superClass != null && !superClass.isAssignableFrom(stub.getClass())) {
throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" +
superClass.getName() + "' as is required.");
}
return stub;
}
catch (ClassCastException ccex) {
throw new RuntimeException("The UDF class is not a proper subclass of " + superClass.getName(), ccex);
}
}
private static int[] asArray(List<Integer> list) {
int[] a = new int[list.size()];
int i = 0;
for (int val : list) {
a[i++] = val;
}
return a;
}
}