/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.api.common.operators.base;
import java.io.Serializable;
import java.util.Collections;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.stratosphere.api.common.InvalidProgramException;
import eu.stratosphere.api.common.aggregators.Aggregator;
import eu.stratosphere.api.common.aggregators.AggregatorRegistry;
import eu.stratosphere.api.common.aggregators.ConvergenceCriterion;
import eu.stratosphere.api.common.functions.AbstractFunction;
import eu.stratosphere.api.common.functions.GenericCollectorMap;
import eu.stratosphere.api.common.operators.IterationOperator;
import eu.stratosphere.api.common.operators.Operator;
import eu.stratosphere.api.common.operators.OperatorInformation;
import eu.stratosphere.api.common.operators.SingleInputOperator;
import eu.stratosphere.api.common.operators.UnaryOperatorInformation;
import eu.stratosphere.api.common.operators.util.UserCodeClassWrapper;
import eu.stratosphere.api.common.operators.util.UserCodeWrapper;
import eu.stratosphere.configuration.Configuration;
import eu.stratosphere.types.LongValue;
import eu.stratosphere.types.Nothing;
import eu.stratosphere.types.NothingTypeInfo;
import eu.stratosphere.util.Collector;
import eu.stratosphere.util.Visitor;
/**
*
*/
public class BulkIterationBase<T> extends SingleInputOperator<T, T, AbstractFunction> implements IterationOperator {
private static String DEFAULT_NAME = "<Unnamed Bulk Iteration>";
public static final String TERMINATION_CRITERION_AGGREGATOR_NAME = "terminationCriterion.aggregator";
private Operator<T> iterationResult;
private final Operator<T> inputPlaceHolder;
private final AggregatorRegistry aggregators = new AggregatorRegistry();
private int numberOfIterations = -1;
protected Operator<?> terminationCriterion;
// --------------------------------------------------------------------------------------------
/**
*
*/
public BulkIterationBase(UnaryOperatorInformation<T, T> operatorInfo) {
this(operatorInfo, DEFAULT_NAME);
}
/**
* @param name
*/
public BulkIterationBase(UnaryOperatorInformation<T, T> operatorInfo, String name) {
super(new UserCodeClassWrapper<AbstractFunction>(AbstractFunction.class), operatorInfo, name);
inputPlaceHolder = new PartialSolutionPlaceHolder<T>(this, this.getOperatorInfo());
}
// --------------------------------------------------------------------------------------------
/**
* @return The operator representing the partial solution.
*/
public Operator<T> getPartialSolution() {
return this.inputPlaceHolder;
}
/**
* @param result
*/
public void setNextPartialSolution(Operator<T> result) {
if (result == null) {
throw new NullPointerException("Operator producing the next partial solution must not be null.");
}
this.iterationResult = result;
}
/**
* @return The operator representing the next partial solution.
*/
public Operator<T> getNextPartialSolution() {
return this.iterationResult;
}
/**
* @return The operator representing the termination criterion.
*/
public Operator<?> getTerminationCriterion() {
return this.terminationCriterion;
}
/**
* @param criterion
*/
public <X> void setTerminationCriterion(Operator<X> criterion) {
CollectorMapOperatorBase<X, Nothing, TerminationCriterionMapper<X>> mapper =
new CollectorMapOperatorBase<X, Nothing, TerminationCriterionMapper<X>>(
new TerminationCriterionMapper<X>(),
new UnaryOperatorInformation<X, Nothing>(criterion.getOperatorInfo().getOutputType(), new NothingTypeInfo()),
"Termination Criterion Aggregation Wrapper");
mapper.setInput(criterion);
this.terminationCriterion = mapper;
this.getAggregators().registerAggregationConvergenceCriterion(TERMINATION_CRITERION_AGGREGATOR_NAME, TerminationCriterionAggregator.class, TerminationCriterionAggregationConvergence.class);
}
/**
* @param num
*/
public void setMaximumNumberOfIterations(int num) {
if (num < 1) {
throw new IllegalArgumentException("The number of iterations must be at least one.");
}
this.numberOfIterations = num;
}
public int getMaximumNumberOfIterations() {
return this.numberOfIterations;
}
@Override
public AggregatorRegistry getAggregators() {
return this.aggregators;
}
/**
* @throws Exception
*/
public void validate() throws InvalidProgramException {
if (this.input == null) {
throw new RuntimeException("Operator for initial partial solution is not set.");
}
if (this.iterationResult == null) {
throw new InvalidProgramException("Operator producing the next version of the partial " +
"solution (iteration result) is not set.");
}
if (this.terminationCriterion == null && this.numberOfIterations <= 0) {
throw new InvalidProgramException("No termination condition is set " +
"(neither fix number of iteration nor termination criterion).");
}
}
/**
* The BulkIteration meta operator cannot have broadcast inputs.
*
* @return An empty map.
*/
public Map<String, Operator<?>> getBroadcastInputs() {
return Collections.emptyMap();
}
/**
* The BulkIteration meta operator cannot have broadcast inputs.
* This method always throws an exception.
*
* @param name Ignored.
* @param root Ignored.
*/
public void setBroadcastVariable(String name, Operator<?> root) {
throw new UnsupportedOperationException("The BulkIteration meta operator cannot have broadcast inputs.");
}
/**
* The BulkIteration meta operator cannot have broadcast inputs.
* This method always throws an exception.
*
* @param inputs Ignored
*/
public <X> void setBroadcastVariables(Map<String, Operator<X>> inputs) {
throw new UnsupportedOperationException("The BulkIteration meta operator cannot have broadcast inputs.");
}
// --------------------------------------------------------------------------------------------
/**
* Specialized operator to use as a recognizable place-holder for the input to the
* step function when composing the nested data flow.
*/
public static class PartialSolutionPlaceHolder<OT> extends Operator<OT> {
private final BulkIterationBase<OT> containingIteration;
public PartialSolutionPlaceHolder(BulkIterationBase<OT> container, OperatorInformation<OT> operatorInfo) {
super(operatorInfo, "Partial Solution");
this.containingIteration = container;
}
public BulkIterationBase<OT> getContainingBulkIteration() {
return this.containingIteration;
}
@Override
public void accept(Visitor<Operator<?>> visitor) {
visitor.preVisit(this);
visitor.postVisit(this);
}
@Override
public UserCodeWrapper<?> getUserCodeWrapper() {
return null;
}
}
/**
* Special Mapper that is added before a termination criterion and is only a container for an special aggregator
*/
public static class TerminationCriterionMapper<X> extends AbstractFunction implements Serializable, GenericCollectorMap<X, Nothing> {
private static final long serialVersionUID = 1L;
private TerminationCriterionAggregator aggregator;
@Override
public void open(Configuration parameters) {
aggregator = getIterationRuntimeContext().getIterationAggregator(TERMINATION_CRITERION_AGGREGATOR_NAME);
}
@Override
public void map(X in, Collector<Nothing> out) {
aggregator.aggregate(1L);
}
}
/**
* Aggregator that basically only adds 1 for every output tuple of the termination criterion branch
*/
public static class TerminationCriterionAggregator implements Aggregator<LongValue> {
private long count;
@Override
public LongValue getAggregate() {
return new LongValue(count);
}
public void aggregate(long count) {
this.count += count;
}
@Override
public void aggregate(LongValue count) {
this.count += count.getValue();
}
@Override
public void reset() {
count = 0;
}
}
/**
* Convergence for the termination criterion is reached if no tuple is output at current iteration for the termination criterion branch
*/
public static class TerminationCriterionAggregationConvergence implements ConvergenceCriterion<LongValue> {
private static final Log log = LogFactory.getLog(TerminationCriterionAggregationConvergence.class);
@Override
public boolean isConverged(int iteration, LongValue countAggregate) {
long count = countAggregate.getValue();
if (log.isInfoEnabled()) {
log.info("Termination criterion stats in iteration [" + iteration + "]: " + count);
}
if(count == 0) {
return true;
}
else {
return false;
}
}
}
}