package aima.core.learning.neural;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import aima.core.learning.data.DataResource;
import aima.core.learning.framework.DataSet;
import aima.core.learning.framework.Example;
import aima.core.util.Util;
import aima.core.util.datastructure.Pair;
/**
* @author Ravi Mohan
*
*/
public abstract class NNDataSet {
/*
* This class represents a source of examples to the rest of the nn
* framework. Assumes only one function approximator works on an instance at
* a given point in time
*/
/*
* the parsed and preprocessed form of the dataset.
*/
private List<NNExample> dataset;
/*
* a copy from which examples are drawn.
*/
private List<NNExample> presentlyProcessed = new ArrayList<NNExample>();
/*
* list of mean Values for all components of raw data set
*/
private List<Double> means;
/*
* list of stdev Values for all components of raw data set
*/
private List<Double> stdevs;
/*
* the normalized data set
*/
protected List<List<Double>> nds;
/*
* the column numbers of the "target"
*/
protected List<Integer> targetColumnNumbers;
/*
* population delegated to subclass because only subclass knows which
* column(s) is target
*/
public abstract void setTargetColumns();
/*
* create a normalized data "table" from the data in the file. At this
* stage, the data isnot split into input pattern and tragets
*/
public void createNormalizedDataFromFile(String filename) throws Exception {
List<List<Double>> rds = new ArrayList<List<Double>>();
// create raw data set
try (BufferedReader reader = new BufferedReader(new InputStreamReader(
DataResource.class.getResourceAsStream(filename + ".csv")))) {
String line;
while ((line = reader.readLine()) != null) {
rds.add(exampleFromString(line, ","));
}
}
// normalize raw dataset
nds = normalize(rds);
}
/*
* create a normalized data "table" from the DataSet using numerizer. At
* this stage, the data isnot split into input pattern and targets TODO
* remove redundancy of recreating the target columns. the numerizer has
* already isolated the targets
*/
public void createNormalizedDataFromDataSet(DataSet ds, Numerizer numerizer)
throws Exception {
List<List<Double>> rds = rawExamplesFromDataSet(ds, numerizer);
// normalize raw dataset
nds = normalize(rds);
}
/*
* Gets (and removes) a random example from the 'presentlyProcessed'
*/
public NNExample getExampleAtRandom() {
int i = Util.randomNumberBetween(0, (presentlyProcessed.size() - 1));
return presentlyProcessed.remove(i);
}
/*
* Gets (and removes) a random example from the 'presentlyProcessed'
*/
public NNExample getExample(int index) {
return presentlyProcessed.remove(index);
}
/*
* check if any more examples remain to be processed
*/
public boolean hasMoreExamples() {
return presentlyProcessed.size() > 0;
}
/*
* check how many examples remain to be processed
*/
public int howManyExamplesLeft() {
return presentlyProcessed.size();
}
/*
* refreshes the presentlyProcessed dataset so it can be used for a new
* epoch of training.
*/
public void refreshDataset() {
presentlyProcessed = new ArrayList<NNExample>();
for (NNExample e : dataset) {
presentlyProcessed.add(e.copyExample());
}
}
/*
* method called by clients to set up data set and make it ready for
* processing
*/
public void createExamplesFromFile(String filename) throws Exception {
createNormalizedDataFromFile(filename);
setTargetColumns();
createExamples();
}
/*
* method called by clients to set up data set and make it ready for
* processing
*/
public void createExamplesFromDataSet(DataSet ds, Numerizer numerizer)
throws Exception {
createNormalizedDataFromDataSet(ds, numerizer);
setTargetColumns();
createExamples();
}
public List<List<Double>> getNormalizedData() {
return nds;
}
public List<Double> getMeans() {
return means;
}
public List<Double> getStdevs() {
return stdevs;
}
//
// PRIVATE METHODS
//
/*
* create Example instances from a normalized data "table".
*/
private void createExamples() {
dataset = new ArrayList<NNExample>();
for (List<Double> dataLine : nds) {
List<Double> input = new ArrayList<Double>();
List<Double> target = new ArrayList<Double>();
for (int i = 0; i < dataLine.size(); i++) {
if (targetColumnNumbers.contains(i)) {
target.add(dataLine.get(i));
} else {
input.add(dataLine.get(i));
}
}
dataset.add(new NNExample(input, target));
}
refreshDataset();// to populate the preentlyProcessed dataset
}
private List<List<Double>> normalize(List<List<Double>> rds) {
int rawDataLength = rds.get(0).size();
List<List<Double>> nds = new ArrayList<List<Double>>();
means = new ArrayList<Double>();
stdevs = new ArrayList<Double>();
List<List<Double>> normalizedColumns = new ArrayList<List<Double>>();
// clculate means for each coponent of example data
for (int i = 0; i < rawDataLength; i++) {
List<Double> columnValues = new ArrayList<Double>();
for (List<Double> rawDatum : rds) {
columnValues.add(rawDatum.get(i));
}
double mean = Util.calculateMean(columnValues);
means.add(mean);
double stdev = Util.calculateStDev(columnValues, mean);
stdevs.add(stdev);
normalizedColumns.add(Util.normalizeFromMeanAndStdev(columnValues,
mean, stdev));
}
// re arrange data from columns
// TODO Assert normalized columns have same size etc
int columnLength = normalizedColumns.get(0).size();
int numberOfColumns = normalizedColumns.size();
for (int i = 0; i < columnLength; i++) {
List<Double> lst = new ArrayList<Double>();
for (int j = 0; j < numberOfColumns; j++) {
lst.add(normalizedColumns.get(j).get(i));
}
nds.add(lst);
}
return nds;
}
private List<Double> exampleFromString(String line, String separator) {
// assumes all values for inout and target are doubles
List<Double> rexample = new ArrayList<Double>();
List<String> attributeValues = Arrays.asList(line.split(separator));
for (String valString : attributeValues) {
rexample.add(Double.parseDouble(valString));
}
return rexample;
}
private List<List<Double>> rawExamplesFromDataSet(DataSet ds,
Numerizer numerizer) {
// assumes all values for inout and target are doubles
List<List<Double>> rds = new ArrayList<List<Double>>();
for (int i = 0; i < ds.size(); i++) {
List<Double> rexample = new ArrayList<Double>();
Example e = ds.getExample(i);
Pair<List<Double>, List<Double>> p = numerizer.numerize(e);
List<Double> attributes = p.getFirst();
for (Double d : attributes) {
rexample.add(d);
}
List<Double> targets = p.getSecond();
for (Double d : targets) {
rexample.add(d);
}
rds.add(rexample);
}
return rds;
}
}