/*
* RapidMiner
*
* Copyright (C) 2001-2008 by Rapid-I and the contributors
*
* Complete list of developers available at our web site:
*
* http://rapid-i.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
package com.rapidminer.operator.preprocessing.filter;
import java.util.Iterator;
import java.util.List;
import com.rapidminer.example.Attribute;
import com.rapidminer.example.Example;
import com.rapidminer.example.ExampleSet;
import com.rapidminer.operator.IOObject;
import com.rapidminer.operator.Operator;
import com.rapidminer.operator.OperatorDescription;
import com.rapidminer.operator.OperatorException;
import com.rapidminer.operator.UserError;
import com.rapidminer.parameter.ParameterType;
import com.rapidminer.parameter.ParameterTypeBoolean;
/**
* This operator generates TF-IDF values from the input data. The input example
* set must contain either simple counts, which will be normalized during
* calculation of the term frequency TF, or it already contains the calculated
* term frequency values (in this case no normalization will be done).
*
* @author Ingo Mierswa
* @version $Id: TFIDFFilter.java,v 1.7 2008/07/07 07:06:40 ingomierswa Exp $
*/
public class TFIDFFilter extends Operator {
/** The parameter name for "Indicates if term frequency values should be generated (must be done if input data is given as simple occurence counts)." */
public static final String PARAMETER_CALCULATE_TERM_FREQUENCIES = "calculate_term_frequencies";
private static final Class[] INPUT_CLASSES = { ExampleSet.class };
private static final Class[] OUTPUT_CLASSES = { ExampleSet.class };
public Class<?>[] getInputClasses() {
return INPUT_CLASSES;
}
public Class<?>[] getOutputClasses() {
return OUTPUT_CLASSES;
}
public TFIDFFilter(OperatorDescription description) {
super(description);
}
public IOObject[] apply() throws OperatorException {
ExampleSet exampleSet = getInput(ExampleSet.class);
if (exampleSet.size() < 1)
throw new UserError(this, 110, new Object[] { "1" });
if (exampleSet.getAttributes().size() == 0)
throw new UserError(this, 106, new Object[0]);
for (Attribute attribute : exampleSet.getAttributes()) {
if (!attribute.isNumerical())
throw new UserError(this, 104, new Object[] { getName(), attribute.getName() });
}
// init
double[] termFrequencySum = new double[exampleSet.size()];
int[] documentFrequencies = new int[exampleSet.getAttributes().size()];
// calculate frequencies
int exampleCounter = 0;
Iterator<Example> reader = exampleSet.iterator();
while (reader.hasNext()) {
Example example = reader.next();
int i = 0;
for (Attribute attribute : exampleSet.getAttributes()) {
double value = example.getValue(attribute);
termFrequencySum[exampleCounter] += value;
if (value > 0)
documentFrequencies[i]++;
i++;
}
exampleCounter++;
checkForStop();
}
// calculate IDF values
double[] inverseDocumentFrequencies = new double[documentFrequencies.length];
for (int i = 0; i < exampleSet.getAttributes().size(); i++)
inverseDocumentFrequencies[i] = Math.log((double) exampleSet.size() / (double) documentFrequencies[i]);
// set values
boolean calculateTermFrequencies = getParameterAsBoolean(PARAMETER_CALCULATE_TERM_FREQUENCIES);
exampleCounter = 0;
reader = exampleSet.iterator();
while (reader.hasNext()) {
Example example = reader.next();
int i = 0;
for (Attribute attribute : exampleSet.getAttributes()) {
double value = example.getValue(attribute);
if (termFrequencySum[exampleCounter] == 0.0d) {
example.setValue(attribute, 0.0d);
} else {
double tf = value;
if (calculateTermFrequencies)
tf /= termFrequencySum[exampleCounter];
double idf = inverseDocumentFrequencies[i];
example.setValue(attribute, (tf * idf));
}
i++;
}
exampleCounter++;
checkForStop();
}
return new IOObject[] { exampleSet };
}
public List<ParameterType> getParameterTypes() {
List<ParameterType> types = super.getParameterTypes();
ParameterType type = new ParameterTypeBoolean(PARAMETER_CALCULATE_TERM_FREQUENCIES, "Indicates if term frequency values should be generated (must be done if input data is given as simple occurence counts).", true);
type.setExpert(false);
types.add(type);
return types;
}
}