/*
* RapidMiner
*
* Copyright (C) 2001-2011 by Rapid-I and the contributors
*
* Complete list of developers available at our web site:
*
* http://rapid-i.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
package com.rapidminer.operator.validation;
import java.util.List;
import com.rapidminer.example.ExampleSet;
import com.rapidminer.example.set.MappedExampleSet;
import com.rapidminer.operator.OperatorCapability;
import com.rapidminer.operator.OperatorDescription;
import com.rapidminer.operator.OperatorException;
import com.rapidminer.operator.ValueDouble;
import com.rapidminer.operator.ports.metadata.MDInteger;
import com.rapidminer.operator.visualization.ProcessLogOperator;
import com.rapidminer.parameter.ParameterType;
import com.rapidminer.parameter.ParameterTypeBoolean;
import com.rapidminer.parameter.ParameterTypeDouble;
import com.rapidminer.parameter.ParameterTypeInt;
import com.rapidminer.parameter.UndefinedParameterError;
import com.rapidminer.tools.RandomGenerator;
/**
* <p>This validation operator performs several bootstrapped samplings (sampling with replacement)
* on the input set and trains a model on these samples. The remaining samples, i.e. those which
* were not sampled, build a test set on which the model is evaluated. This process is repeated
* for the specified number of iterations after which the average performance is calculated.</p>
*
* <p>The basic setup is the same as for the usual cross validation operator. The first inner
* operator must provide a model and the second a performance vector. Please note that this operator
* does not regard example weights, i.e. weights specified in a weight column.</p>
*
* <p>This validation operator provides several values which can be logged
* by means of a {@link ProcessLogOperator}. All performance estimation operators
* of RapidMiner provide access to the average values calculated during the estimation.
* Since the operator cannot ensure the names of the delivered criteria, the
* ProcessLog operator can access the values via the generic value names:</p>
* <ul>
* <li>performance: the value for the main criterion calculated by this validation operator</li>
* <li>performance1: the value of the first criterion of the performance vector calculated</li>
* <li>performance2: the value of the second criterion of the performance vector calculated</li>
* <li>performance3: the value of the third criterion of the performance vector calculated</li>
* <li>for the main criterion, also the variance and the standard deviation can be
* accessed where applicable.</li>
* </ul>
*
* @author Ingo Mierswa, Tobias Malbrecht
*/
public class BootstrappingValidation extends ValidationChain {
public static final String PARAMETER_NUMBER_OF_VALIDATIONS = "number_of_validations";
public static final String PARAMETER_SAMPLE_RATIO = "sample_ratio";
public static final String PARAMETER_USE_WEIGHTS = "use_weights";
public static final String PARAMETER_AVERAGE_PERFORMANCES_ONLY = "average_performances_only";
private int number;
private int iteration;
public BootstrappingValidation(OperatorDescription description) {
super(description);
addValue(new ValueDouble("iteration", "The number of the current iteration.") {
@Override
public double getDoubleValue() {
return iteration;
}
});
}
@Override
public void estimatePerformance(ExampleSet inputSet) throws OperatorException {
number = getParameterAsInt(PARAMETER_NUMBER_OF_VALIDATIONS);
int size = (int) Math.round(inputSet.size() * getParameterAsDouble(PARAMETER_SAMPLE_RATIO));
// start bootstrapping loop
RandomGenerator random = RandomGenerator.getRandomGenerator(this);
for (iteration = 0; iteration < number; iteration++) {
int[] mapping = null;
if (getParameterAsBoolean(PARAMETER_USE_WEIGHTS) && inputSet.getAttributes().getWeight() != null) {
mapping = MappedExampleSet.createWeightedBootstrappingMapping(inputSet, size, random);
} else {
mapping = MappedExampleSet.createBootstrappingMapping(inputSet, size, random);
}
MappedExampleSet trainingSet = new MappedExampleSet((ExampleSet)inputSet.clone(), mapping, true);
learn(trainingSet);
MappedExampleSet inverseExampleSet = new MappedExampleSet((ExampleSet)inputSet.clone(), mapping, false);
evaluate(inverseExampleSet);
inApplyLoop();
}
// end loop
}
@Override
protected MDInteger getTestSetSize(MDInteger originalSize) throws UndefinedParameterError {
return originalSize.multiply(1d - getParameterAsDouble(PARAMETER_SAMPLE_RATIO));
}
@Override
protected MDInteger getTrainingSetSize(MDInteger originalSize) throws UndefinedParameterError {
return originalSize.multiply(getParameterAsDouble(PARAMETER_SAMPLE_RATIO));
}
@Override
public List<ParameterType> getParameterTypes() {
List<ParameterType> types = super.getParameterTypes();
ParameterType type = new ParameterTypeInt(PARAMETER_NUMBER_OF_VALIDATIONS, "The number of validations that should be executed.", 2, Integer.MAX_VALUE, 10);
type.setExpert(false);
types.add(type);
types.add(new ParameterTypeDouble(PARAMETER_SAMPLE_RATIO, "This ratio of examples will be sampled (with replacement) in each iteration.", 0.0d, Double.POSITIVE_INFINITY, 1.0d));
types.add(new ParameterTypeBoolean(PARAMETER_USE_WEIGHTS, "If checked, example weights will be used for bootstrapping if such weights are available.", true));
types.add(new ParameterTypeBoolean(PARAMETER_AVERAGE_PERFORMANCES_ONLY, "Indicates if only performance vectors should be averaged or all types of averagable result vectors.", true));
types.addAll(RandomGenerator.getRandomGeneratorParameters(this));
return types;
}
@Override
public boolean supportsCapability(OperatorCapability capability) {
return true;
}
}