/*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* MathExpression.java
* Copyright (C) 2004 Prados Julien
* Copyright (C) 2002-2012 University of Waikato, Hamilton, New Zealand
*/
package weka.filters.unsupervised.attribute;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Vector;
import weka.core.AttributeStats;
import weka.core.Capabilities;
import weka.core.Capabilities.Capability;
import weka.core.DenseInstance;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.MathematicalExpression;
import weka.core.Option;
import weka.core.Range;
import weka.core.RevisionUtils;
import weka.core.SparseInstance;
import weka.core.Utils;
import weka.filters.UnsupervisedFilter;
/**
* <!-- globalinfo-start --> Modify numeric attributes according to a given expression
* <p/>
* <!-- globalinfo-end -->
*
* <!-- options-start --> Valid options are:
* <p/>
*
* <pre>
* -unset-class-temporarily
* Unsets the class index temporarily before the filter is
* applied to the data.
* (default: no)
* </pre>
*
* <pre>
* -E <expression>
* Specify the expression to apply. Eg. pow(A,6)/(MEAN+MAX)
* Supported operators are +, -, *, /, pow, log,
* abs, cos, exp, sqrt, tan, sin, ceil, floor, rint, (, ),
* MEAN, MAX, MIN, SD, COUNT, SUM, SUMSQUARED, ifelse. The 'A'
* letter refers to the value of the attribute being processed.
* Other attribute values (numeric only) can be accessed through
* the variables A1, A2, A3, ...
* </pre>
*
* <pre>
* -R <index1,index2-index4,...>
* Specify list of columns to ignore. First and last are valid
* indexes. (default none)
* </pre>
*
* <pre>
* -V
* Invert matching sense (i.e. only modify specified columns)
* </pre>
*
* <!-- options-end -->
*
* @author Eibe Frank (eibe@cs.waikato.ac.nz)
* @author Prados Julien (julien.prados@cui.unige.ch)
* @version $Revision: 8034 $
* @see MathematicalExpression
*/
public class MathExpression extends PotentialClassIgnorer implements UnsupervisedFilter {
/** for serialization */
static final long serialVersionUID = -3713222714671997901L;
/** Stores which columns to select as a funky range */
protected Range m_SelectCols = new Range();
/** The default modification expression */
public static final String m_defaultExpression = "(A-MIN)/(MAX-MIN)";
/** The modification expression */
private String m_expression = m_defaultExpression;
/** Attributes statistics */
private AttributeStats[] m_attStats;
/**
* Constructor
*/
public MathExpression() {
super();
setInvertSelection(false);
}
/**
* Returns a string describing this filter
*
* @return a description of the filter suitable for displaying in the explorer/experimenter gui
*/
public String globalInfo() {
return "Modify numeric attributes according to a given expression ";
}
/**
* Returns the Capabilities of this filter.
*
* @return the capabilities of this object
* @see Capabilities
*/
public Capabilities getCapabilities() {
Capabilities result = super.getCapabilities();
result.disableAll();
// attributes
result.enableAllAttributes();
result.enable(Capability.MISSING_VALUES);
// class
result.enableAllClasses();
result.enable(Capability.MISSING_CLASS_VALUES);
result.enable(Capability.NO_CLASS);
return result;
}
/**
* Sets the format of the input instances.
*
* @param instanceInfo
* an Instances object containing the input instance structure (any instances contained in the object are ignored - only the structure is required).
* @return true if the outputFormat may be collected immediately
* @throws Exception
* if the input format can't be set successfully
*/
public boolean setInputFormat(Instances instanceInfo) throws Exception {
m_SelectCols.setUpper(instanceInfo.numAttributes() - 1);
super.setInputFormat(instanceInfo);
setOutputFormat(instanceInfo);
m_attStats = null;
return true;
}
/**
* Input an instance for filtering. Filter requires all training instances be read before producing output.
*
* @param instance
* the input instance
* @return true if the filtered instance may now be collected with output().
* @throws IllegalStateException
* if no input format has been set.
*/
public boolean input(Instance instance) throws Exception {
if (getInputFormat() == null) {
throw new IllegalStateException("No input instance format defined");
}
if (m_NewBatch) {
resetQueue();
m_NewBatch = false;
}
if (m_attStats == null) {
bufferInput(instance);
return false;
} else {
convertInstance(instance);
return true;
}
}
/**
* Signify that this batch of input to the filter is finished. If the filter requires all instances prior to filtering, output() may now be called to retrieve the filtered instances.
*
* @return true if there are instances pending output
* @throws IllegalStateException
* if no input structure has been defined
*/
public boolean batchFinished() throws Exception {
if (getInputFormat() == null) {
throw new IllegalStateException("No input instance format defined");
}
if (m_attStats == null) {
Instances input = getInputFormat();
m_attStats = new AttributeStats[input.numAttributes()];
for (int i = 0; i < input.numAttributes(); i++) {
if (input.attribute(i).isNumeric() && (input.classIndex() != i)) {
m_attStats[i] = input.attributeStats(i);
}
}
// Convert pending input instances
for (int i = 0; i < input.numInstances(); i++) {
convertInstance(input.instance(i));
}
}
// Free memory
flushInput();
m_NewBatch = true;
return (numPendingOutput() != 0);
}
/**
* Evaluates the symbols.
*
* @param symbols
* the symbols to use for evaluation
* @return the calculated value, Double.NaN in case of an error
*/
protected double eval(HashMap symbols) {
/*SymbolFactory sf;
ByteArrayInputStream parserInput;
Parser parser;
double result;
try {
sf = new DefaultSymbolFactory();
parserInput = new ByteArrayInputStream(m_expression.getBytes());
parser = new Parser(new Scanner(parserInput, sf), sf);
parser.setSymbols(symbols);
parser.parse();
result = parser.getResult();
}
catch (Exception e) {
result = Double.NaN;
e.printStackTrace();
}
return result;*/
return 0.0;
}
/**
* Convert a single instance over. The converted instance is added to the end of the output queue.
*
* @param instance
* the instance to convert
* @throws Exception
* if instance cannot be converted
*/
private void convertInstance(Instance instance) throws Exception {
Instance inst = null;
HashMap symbols = new HashMap(5);
if (instance instanceof SparseInstance) {
double[] newVals = new double[instance.numAttributes()];
int[] newIndices = new int[instance.numAttributes()];
double[] vals = instance.toDoubleArray();
double[] valsCopy = instance.toDoubleArray();
// add a symbol for all the numeric attributes except the class
for (int z = 0; z < getInputFormat().numAttributes(); z++) {
if (instance.attribute(z).isNumeric() && z != getInputFormat().classIndex()) {
symbols.put("A" + (z + 1), new Double(valsCopy[z]));
}
}
int ind = 0;
double value;
for (int j = 0; j < instance.numAttributes(); j++) {
if (m_SelectCols.isInRange(j)) {
if (instance.attribute(j).isNumeric() && (!Utils.isMissingValue(vals[j])) && (getInputFormat().classIndex() != j)) {
symbols.put("A", new Double(vals[j]));
symbols.put("MAX", new Double(m_attStats[j].numericStats.max));
symbols.put("MIN", new Double(m_attStats[j].numericStats.min));
symbols.put("MEAN", new Double(m_attStats[j].numericStats.mean));
symbols.put("SD", new Double(m_attStats[j].numericStats.stdDev));
symbols.put("COUNT", new Double(m_attStats[j].numericStats.count));
symbols.put("SUM", new Double(m_attStats[j].numericStats.sum));
symbols.put("SUMSQUARED", new Double(m_attStats[j].numericStats.sumSq));
value = eval(symbols);
if (Double.isNaN(value) || Double.isInfinite(value)) {
System.err.println("WARNING:Error in evaluating the expression: missing value set");
value = Utils.missingValue();
}
if (value != 0.0) {
newVals[ind] = value;
newIndices[ind] = j;
ind++;
}
}
} else {
value = vals[j];
if (value != 0.0) {
newVals[ind] = value;
newIndices[ind] = j;
ind++;
}
}
}
double[] tempVals = new double[ind];
int[] tempInd = new int[ind];
System.arraycopy(newVals, 0, tempVals, 0, ind);
System.arraycopy(newIndices, 0, tempInd, 0, ind);
inst = new SparseInstance(instance.weight(), tempVals, tempInd, instance.numAttributes());
} else {
double[] vals = instance.toDoubleArray();
double[] valsCopy = instance.toDoubleArray();
// add a symbol for all the numeric attributes except the class
for (int z = 0; z < getInputFormat().numAttributes(); z++) {
if (instance.attribute(z).isNumeric() && z != getInputFormat().classIndex()) {
symbols.put("A" + (z + 1), new Double(valsCopy[z]));
}
}
for (int j = 0; j < getInputFormat().numAttributes(); j++) {
if (m_SelectCols.isInRange(j)) {
if (instance.attribute(j).isNumeric() && (!Utils.isMissingValue(vals[j])) && (getInputFormat().classIndex() != j)) {
symbols.put("A", new Double(vals[j]));
symbols.put("MAX", new Double(m_attStats[j].numericStats.max));
symbols.put("MIN", new Double(m_attStats[j].numericStats.min));
symbols.put("MEAN", new Double(m_attStats[j].numericStats.mean));
symbols.put("SD", new Double(m_attStats[j].numericStats.stdDev));
symbols.put("COUNT", new Double(m_attStats[j].numericStats.count));
symbols.put("SUM", new Double(m_attStats[j].numericStats.sum));
symbols.put("SUMSQUARED", new Double(m_attStats[j].numericStats.sumSq));
vals[j] = eval(symbols);
if (Double.isNaN(vals[j]) || Double.isInfinite(vals[j])) {
System.err.println("WARNING:Error in Evaluation the Expression: missing value set");
vals[j] = Utils.missingValue();
}
}
}
}
inst = new DenseInstance(instance.weight(), vals);
}
inst.setDataset(instance.dataset());
push(inst);
}
/**
* Parses a given list of options.
* <p/>
*
* <!-- options-start --> Valid options are:
* <p/>
*
* <pre>
* -unset-class-temporarily
* Unsets the class index temporarily before the filter is
* applied to the data.
* (default: no)
* </pre>
*
* <pre>
* -E <expression>
* Specify the expression to apply. Eg. pow(A,6)/(MEAN+MAX)
* Supported operators are +, -, *, /, pow, log,
* abs, cos, exp, sqrt, tan, sin, ceil, floor, rint, (, ),
* MEAN, MAX, MIN, SD, COUNT, SUM, SUMSQUARED, ifelse. The 'A'
* letter refers to the value of the attribute being processed.
* Other attribute values (numeric only) can be accessed through
* the variables A1, A2, A3, ...
* </pre>
*
* <pre>
* -R <index1,index2-index4,...>
* Specify list of columns to ignore. First and last are valid
* indexes. (default none)
* </pre>
*
* <pre>
* -V
* Invert matching sense (i.e. only modify specified columns)
* </pre>
*
* <!-- options-end -->
*
* @param options
* the list of options as an array of strings
* @throws Exception
* if an option is not supported
*/
public void setOptions(String[] options) throws Exception {
super.setOptions(options);
String expString = Utils.getOption('E', options);
if (expString.length() != 0) {
setExpression(expString);
} else {
setExpression(m_defaultExpression);
}
String ignoreList = Utils.getOption('R', options);
if (ignoreList.length() != 0) {
setIgnoreRange(ignoreList);
}
setInvertSelection(Utils.getFlag('V', options));
}
/**
* Gets the current settings of the filter.
*
* @return an array of strings suitable for passing to setOptions
*/
public String[] getOptions() {
Vector result;
String[] options;
int i;
result = new Vector();
options = super.getOptions();
for (i = 0; i < options.length; i++)
result.add(options[i]);
result.add("-E");
result.add(getExpression());
if (getInvertSelection())
result.add("-V");
if (!getIgnoreRange().equals("")) {
result.add("-R");
result.add(getIgnoreRange());
}
return (String[]) result.toArray(new String[result.size()]);
}
/**
* Returns an enumeration describing the available options.
*
* @return an enumeration of all the available options.
*/
public Enumeration listOptions() {
Vector result = new Vector();
Enumeration enm = super.listOptions();
while (enm.hasMoreElements())
result.add(enm.nextElement());
result.addElement(new Option("\tSpecify the expression to apply. Eg. pow(A,6)/(MEAN+MAX)" + "\n\tSupported operators are +, -, *, /, pow, log," + "\n\tabs, cos, exp, sqrt, tan, sin, ceil, floor, rint, (, ), " + "\n\tMEAN, MAX, MIN, SD, COUNT, SUM, SUMSQUARED, ifelse. The 'A'" + "\n\tletter refers to the value of the attribute being processed." + "\n\tOther attribute values (numeric only) can be accessed through" + "\n\tthe variables A1, A2, A3, ...", "E", 1, "-E <expression>"));
result.addElement(new Option("\tSpecify list of columns to ignore. First and last are valid\n" + "\tindexes. (default none)", "R", 1, "-R <index1,index2-index4,...>"));
result.addElement(new Option("\tInvert matching sense (i.e. only modify specified columns)", "V", 0, "-V"));
return result.elements();
}
/**
* Returns the tip text for this property
*
* @return tip text for this property suitable for displaying in the explorer/experimenter gui
*/
public String expressionTipText() {
return "Specify the expression to apply. The 'A' letter" + "refers to the value of the attribute being processed. " + "MIN,MAX,MEAN,SD" + "refer respectively to minimum, maximum, mean and" + "standard deviation of the attribute being processed. " + "Other attribute values (numeric only) can be accessed " + "through the variables A1, A2, A3, ..." + "\n\tSupported operators are +, -, *, /, pow, log," + "abs, cos, exp, sqrt, tan, sin, ceil, floor, rint, (, )," + "A,MEAN, MAX, MIN, SD, COUNT, SUM, SUMSQUARED, ifelse" + "\n\tEg. pow(A,6)/(MEAN+MAX)*ifelse(A<0,0,sqrt(A))+ifelse(![A>9 && A<15])";
}
/**
* Set the expression to apply
*
* @param expr
* a mathematical expression to apply
*/
public void setExpression(String expr) {
m_expression = expr;
}
/**
* Get the expression
*
* @return the expression
*/
public String getExpression() {
return m_expression;
}
/**
* Returns the tip text for this property
*
* @return tip text for this property suitable for displaying in the explorer/experimenter gui
*/
public String invertSelectionTipText() {
return "Determines whether action is to select or unselect." + " If set to true, only the specified attributes will be modified;" + " If set to false, specified attributes will not be modified.";
}
/**
* Get whether the supplied columns are to be select or unselect
*
* @return true if the supplied columns will be kept
*/
public boolean getInvertSelection() {
return !m_SelectCols.getInvert();
}
/**
* Set whether selected columns should be select or unselect. If true the selected columns are modified. If false the selected columns are not modified.
*
* @param invert
* the new invert setting
*/
public void setInvertSelection(boolean invert) {
m_SelectCols.setInvert(!invert);
}
/**
* Returns the tip text for this property
*
* @return tip text for this property suitable for displaying in the explorer/experimenter gui
*/
public String ignoreRangeTipText() {
return "Specify range of attributes to act on." + " This is a comma separated list of attribute indices, with" + " \"first\" and \"last\" valid values. Specify an inclusive" + " range with \"-\". E.g: \"first-3,5,6-10,last\".";
}
/**
* Get the current range selection.
*
* @return a string containing a comma separated list of ranges
*/
public String getIgnoreRange() {
return m_SelectCols.getRanges();
}
/**
* Set which attributes are to be ignored
*
* @param rangeList
* a string representing the list of attributes. Since the string will typically come from a user, attributes are indexed from 1. <br/>
* eg: first-3,5,6-last
*/
public void setIgnoreRange(String rangeList) {
m_SelectCols.setRanges(rangeList);
}
/**
* Returns the revision string.
*
* @return the revision
*/
public String getRevision() {
return RevisionUtils.extract("$Revision: 8034 $");
}
/**
* Main method for testing this class.
*
* @param argv
* should contain arguments to the filter: use -h for help
*/
public static void main(String[] argv) {
runFilter(new MathExpression(), argv);
}
}