/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
/*
* C45Loader.java
* Copyright (C) 2000 University of Waikato, Hamilton, New Zealand
*
*/
package weka.core.converters;
import weka.core.Attribute;
import weka.core.Instance;
import weka.core.DenseInstance;
import weka.core.Instances;
import weka.core.RevisionUtils;
import weka.core.Utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StreamTokenizer;
import java.util.ArrayList;
/**
<!-- globalinfo-start -->
* Reads a file that is C45 format. Can take a filestem or filestem with .names or .data appended. Assumes that path/<filestem>.names and path/<filestem>.data exist and contain the names and data respectively.
* <p/>
<!-- globalinfo-end -->
*
* @author Mark Hall (mhall@cs.waikato.ac.nz)
* @version $Revision: 5987 $
* @see Loader
*/
public class C45Loader
extends AbstractFileLoader
implements BatchConverter, IncrementalConverter {
/** for serialization */
static final long serialVersionUID = 5454329403218219L;
/** the file extension */
public static String FILE_EXTENSION = ".names";
/**
* Describe variable <code>m_sourceFileData</code> here.
*/
private File m_sourceFileData = null;
/**
* Reader for names file
*/
private transient Reader m_namesReader = null;
/**
* Reader for data file
*/
private transient Reader m_dataReader = null;
/**
* Holds the filestem.
*/
private String m_fileStem;
/**
* Number of attributes in the data (including ignore and label attributes).
*/
private int m_numAttribs;
/**
* Which attributes are ignore or label. These are *not* included in the
* arff representation.
*/
private boolean [] m_ignore;
/**
* Returns a string describing this attribute evaluator
* @return a description of the evaluator suitable for
* displaying in the explorer/experimenter gui
*/
public String globalInfo() {
return "Reads a file that is C45 format. Can take a filestem or filestem "
+"with .names or .data appended. Assumes that path/<filestem>.names and "
+"path/<filestem>.data exist and contain the names and data "
+"respectively.";
}
/**
* Resets the Loader ready to read a new data set or the
* same data set again.
*
* @throws IOException if something goes wrong
*/
public void reset() throws IOException {
m_structure = null;
setRetrieval(NONE);
if (m_File != null) {
setFile(new File(m_File));
}
}
/**
* Get the file extension used for arff files
*
* @return the file extension
*/
public String getFileExtension() {
return FILE_EXTENSION;
}
/**
* Gets all the file extensions used for this type of file
*
* @return the file extensions
*/
public String[] getFileExtensions() {
return new String[]{".names", ".data"};
}
/**
* Returns a description of the file type.
*
* @return a short file description
*/
public String getFileDescription() {
return "C4.5 data files";
}
/**
* Resets the Loader object and sets the source of the data set to be
* the supplied File object.
*
* @param file the source file.
* @exception IOException if an error occurs
*/
public void setSource(File file) throws IOException {
m_structure = null;
setRetrieval(NONE);
if (file == null) {
throw new IOException("Source file object is null!");
}
String fname = file.getName();
String fileStem;
String path = file.getParent();
if (path != null) {
path += File.separator;
} else {
path = "";
}
if (fname.indexOf('.') < 0) {
fileStem = fname;
fname += ".names";
} else {
fileStem = fname.substring(0, fname.lastIndexOf('.'));
fname = fileStem + ".names";
}
m_fileStem = fileStem;
file = new File(path+fname);
m_sourceFile = file;
try {
BufferedReader br = new BufferedReader(new FileReader(file));
m_namesReader = br;
} catch (FileNotFoundException ex) {
throw new IOException("File not found : "+(path+fname));
}
m_sourceFileData = new File(path+fileStem+".data");
try {
BufferedReader br = new BufferedReader(new FileReader(m_sourceFileData));
m_dataReader = br;
} catch (FileNotFoundException ex) {
throw new IOException("File not found : "+(path+fname));
}
m_File = file.getAbsolutePath();
}
/**
* Determines and returns (if possible) the structure (internally the
* header) of the data set as an empty set of instances.
*
* @return the structure of the data set as an empty set of Instances
* @exception IOException if an error occurs
*/
public Instances getStructure() throws IOException {
if (m_sourceFile == null) {
throw new IOException("No source has beenspecified");
}
if (m_structure == null) {
setSource(m_sourceFile);
StreamTokenizer st = new StreamTokenizer(m_namesReader);
initTokenizer(st);
readHeader(st);
}
return m_structure;
}
/**
* Return the full data set. If the structure hasn't yet been determined
* by a call to getStructure then method should do so before processing
* the rest of the data set.
*
* @return the structure of the data set as an empty set of Instances
* @exception IOException if there is no source or parsing fails
*/
public Instances getDataSet() throws IOException {
if (m_sourceFile == null) {
throw new IOException("No source has been specified");
}
if (getRetrieval() == INCREMENTAL) {
throw new IOException("Cannot mix getting Instances in both incremental and batch modes");
}
setRetrieval(BATCH);
if (m_structure == null) {
getStructure();
}
StreamTokenizer st = new StreamTokenizer(m_dataReader);
initTokenizer(st);
// st.ordinaryChar('.');
Instances result = new Instances(m_structure);
Instance current = getInstance(st);
while (current != null) {
result.add(current);
current = getInstance(st);
}
try {
// close the stream
m_dataReader.close();
// reset();
} catch (Exception ex) {
ex.printStackTrace();
}
return result;
}
/**
* Read the data set incrementally---get the next instance in the data
* set or returns null if there are no
* more instances to get. If the structure hasn't yet been
* determined by a call to getStructure then method should do so before
* returning the next instance in the data set.
*
* If it is not possible to read the data set incrementally (ie. in cases
* where the data set structure cannot be fully established before all
* instances have been seen) then an exception should be thrown.
*
* @param structure the dataset header information, will get updated in
* case of string or relational attributes
* @return the next instance in the data set as an Instance object or null
* if there are no more instances to be read
* @exception IOException if there is an error during parsing
*/
public Instance getNextInstance(Instances structure) throws IOException {
if (m_sourceFile == null) {
throw new IOException("No source has been specified");
}
if (getRetrieval() == BATCH) {
throw new IOException("Cannot mix getting Instances in both incremental and batch modes");
}
setRetrieval(INCREMENTAL);
if (m_structure == null) {
getStructure();
}
StreamTokenizer st = new StreamTokenizer(m_dataReader);
initTokenizer(st);
// st.ordinaryChar('.');
Instance nextI = getInstance(st);
if (nextI != null) {
nextI.setDataset(m_structure);
}
else{
try {
// close the stream
m_dataReader.close();
// reset();
} catch (Exception ex) {
ex.printStackTrace();
}
}
return nextI;
}
/**
* Reads an instance using the supplied tokenizer.
*
* @param tokenizer the tokenizer to use
* @return an Instance or null if there are no more instances to read
* @exception IOException if an error occurs
*/
private Instance getInstance(StreamTokenizer tokenizer)
throws IOException {
double [] instance = new double[m_structure.numAttributes()];
ConverterUtils.getFirstToken(tokenizer);
if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
return null;
}
int counter = 0;
for (int i = 0; i < m_numAttribs; i++) {
if (i > 0) {
ConverterUtils.getToken(tokenizer);
}
if (!m_ignore[i]) {
// Check if value is missing.
if (tokenizer.ttype == '?') {
instance[counter++] = Utils.missingValue();
} else {
String val = tokenizer.sval;
if (i == m_numAttribs - 1) {
// remove trailing period
if (val.charAt(val.length()-1) == '.') {
val = val.substring(0,val.length()-1);
}
}
if (m_structure.attribute(counter).isNominal()) {
int index = m_structure.attribute(counter).indexOfValue(val);
if (index == -1) {
ConverterUtils.errms(tokenizer, "nominal value not declared in "
+"header :"+val+" column "+i);
}
instance[counter++] = (double)index;
} else if (m_structure.attribute(counter).isNumeric()) {
try {
instance[counter++] = Double.valueOf(val).doubleValue();
} catch (NumberFormatException e) {
ConverterUtils.errms(tokenizer, "number expected");
}
} else {
System.err.println("Shouldn't get here");
System.exit(1);
}
}
}
}
return new DenseInstance(1.0, instance);
}
/**
* removes the trailing period
*
* @param val the string to work on
* @return the processed string
*/
private String removeTrailingPeriod(String val) {
// remove trailing period
if (val.charAt(val.length()-1) == '.') {
val = val.substring(0,val.length()-1);
}
return val;
}
/**
* Reads header (from the names file) using the supplied tokenizer
*
* @param tokenizer the tokenizer to use
* @exception IOException if an error occurs
*/
private void readHeader(StreamTokenizer tokenizer) throws IOException {
ArrayList<Attribute> attribDefs = new ArrayList<Attribute>();
ArrayList<Integer> ignores = new ArrayList<Integer>();
ConverterUtils.getFirstToken(tokenizer);
if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
ConverterUtils.errms(tokenizer,"premature end of file");
}
m_numAttribs = 1;
// Read the class values
ArrayList<String> classVals = new ArrayList<String>();
while (tokenizer.ttype != StreamTokenizer.TT_EOL) {
String val = tokenizer.sval.trim();
if (val.length() > 0) {
val = removeTrailingPeriod(val);
classVals.add(val);
}
ConverterUtils.getToken(tokenizer);
}
// read the attribute names and types
int counter = 0;
while (tokenizer.ttype != StreamTokenizer.TT_EOF) {
ConverterUtils.getFirstToken(tokenizer);
if (tokenizer.ttype != StreamTokenizer.TT_EOF) {
String attribName = tokenizer.sval;
ConverterUtils.getToken(tokenizer);
if (tokenizer.ttype == StreamTokenizer.TT_EOL) {
ConverterUtils.errms(tokenizer, "premature end of line. Expected "
+"attribute type.");
}
String temp = tokenizer.sval.toLowerCase().trim();
if (temp.startsWith("ignore") || temp.startsWith("label")) {
ignores.add(new Integer(counter));
counter++;
} else if (temp.startsWith("continuous")) {
attribDefs.add(new Attribute(attribName));
counter++;
} else {
counter++;
// read the values of the attribute
ArrayList<String> attribVals = new ArrayList<String>();
while (tokenizer.ttype != StreamTokenizer.TT_EOL &&
tokenizer.ttype != StreamTokenizer.TT_EOF) {
String val = tokenizer.sval.trim();
if (val.length() > 0) {
val = removeTrailingPeriod(val);
attribVals.add(val);
}
ConverterUtils.getToken(tokenizer);
}
attribDefs.add(new Attribute(attribName, attribVals));
}
}
}
boolean ok = true;
int i = -1;
if (classVals.size() == 1) {
// look to see if this is an attribute name (ala c5 names file style)
for (i = 0; i < attribDefs.size(); i++) {
if (((Attribute)attribDefs.get(i))
.name().compareTo((String)classVals.get(0)) == 0) {
ok = false;
m_numAttribs--;
break;
}
}
}
if (ok) {
attribDefs.add(new Attribute("Class", classVals));
}
m_structure = new Instances(m_fileStem, attribDefs, 0);
try {
if (ok) {
m_structure.setClassIndex(m_structure.numAttributes()-1);
} else {
m_structure.setClassIndex(i);
}
} catch (Exception ex) {
ex.printStackTrace();
}
m_numAttribs = m_structure.numAttributes() + ignores.size();
m_ignore = new boolean[m_numAttribs];
for (i = 0; i < ignores.size(); i++) {
m_ignore[((Integer)ignores.get(i)).intValue()] = true;
}
}
/**
* Initializes the stream tokenizer
*
* @param tokenizer the tokenizer to initialize
*/
private void initTokenizer(StreamTokenizer tokenizer) {
tokenizer.resetSyntax();
tokenizer.whitespaceChars(0, (' '-1));
tokenizer.wordChars(' ','\u00FF');
tokenizer.whitespaceChars(',',',');
tokenizer.whitespaceChars(':',':');
// tokenizer.whitespaceChars('.','.');
tokenizer.commentChar('|');
tokenizer.whitespaceChars('\t','\t');
tokenizer.quoteChar('"');
tokenizer.quoteChar('\'');
tokenizer.eolIsSignificant(true);
}
/**
* Returns the revision string.
*
* @return the revision
*/
public String getRevision() {
return RevisionUtils.extract("$Revision: 5987 $");
}
/**
* Main method for testing this class.
*
* @param args should contain <filestem>[.names | data]
*/
public static void main (String [] args) {
runFileLoader(new C45Loader(), args);
}
}