Propagation.java

/*
 * Encog(tm) Core v3.4 - Java Version
 * http://www.heatonresearch.com/encog/
 * https://github.com/encog/encog-java-core
 
 * Copyright 2008-2016 Heaton Research, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *   
 * For more information on Heaton Research copyrights, licenses 
 * and trademarks visit:
 * http://www.heatonresearch.com/copyright
 */
package org.encog.neural.networks.training.propagation;

import java.util.Random;

import org.encog.EncogError;
import org.encog.engine.network.activation.ActivationFunction;
import org.encog.engine.network.activation.ActivationSigmoid;
import org.encog.mathutil.IntRange;
import org.encog.ml.MLMethod;
import org.encog.ml.TrainingImplementationType;
import org.encog.ml.data.MLDataSet;
import org.encog.ml.train.BasicTraining;
import org.encog.neural.error.ErrorFunction;
import org.encog.neural.error.LinearErrorFunction;
import org.encog.neural.flat.FlatNetwork;
import org.encog.neural.networks.ContainsFlat;
import org.encog.neural.networks.training.BatchSize;
import org.encog.neural.networks.training.Train;
import org.encog.util.EncogValidate;
import org.encog.util.EngineArray;
import org.encog.util.concurrency.DetermineWorkload;
import org.encog.util.concurrency.EngineConcurrency;
import org.encog.util.concurrency.MultiThreadable;
import org.encog.util.concurrency.TaskGroup;
import org.encog.util.logging.EncogLogging;

/**
 * Implements basic functionality that is needed by each of the propagation
 * methods. The specifics of each of the propagation methods is implemented
 * inside of the PropagationMethod interface implementors.
 * 
 * @author jheaton
 * 
 */
public abstract class Propagation extends BasicTraining implements Train,
		MultiThreadable, BatchSize, GradientWorkerOwner {

	/**
	 * Used to generate randomness for dropout
	 */
	protected Random dropoutRandomSource = new Random();

	/**
	 * The Dropout rate, between 0 and 1
	 */
	private double dropoutRate = 0;
	
	/**
	 * The current flat network we are using for training, or null for none.
	 */
	private FlatNetwork currentFlatNetwork;

	/**
	 * The number of threads to use.
	 */
	private int numThreads;

	/**
	 * The gradients.
	 */
	protected double[] gradients;

	/**
	 * The last gradients, from the last training iteration.
	 */
	private final double[] lastGradient;

	/**
	 * The network to train.
	 */
	protected final ContainsFlat network;

	/**
	 * The network in indexable form.
	 */
	private final MLDataSet indexable;

	/**
	 * The workers.
	 */
	private GradientWorker[] workers;

	/**
	 * The total error. Used to take the average of.
	 */
	private double totalError;

	/**
	 * Reported exception from the threads.
	 */
	private Throwable reportedException;

	/**
	 * The iteration.
	 */
	private int iteration;

	/**
	 * The flat spot constants.
	 */
	private double[] flatSpot;

	/**
	 * Should we fix flat spots.
	 */
	private boolean shouldFixFlatSpot;

	/**
	 * The error function.
	 */
	private ErrorFunction ef = new LinearErrorFunction();

	/**
	 * The batch size. Specify 1 for pure online training. Specify 0 for pure
	 * batch training (complete training set in one batch). Otherwise specify
	 * the batch size for batch training.
	 */
	private int batchSize = 0;
	
	/**
	 * How much to apply l1 regularization penalty, 0 (default) for none.
	 */
	private double l1;
	
	/**
	 * How much to apply l2 regularization penalty, 0 (default) for none.
	 */
	private double l2;

	private boolean finalized = false;

	/**
	 * Construct a propagation object.
	 * 
	 * @param network
	 *            The network.
	 * @param training
	 *            The training set.
	 */
	public Propagation(final ContainsFlat network, final MLDataSet training) {
		super(TrainingImplementationType.Iterative);
		this.network = network;
		this.currentFlatNetwork = network.getFlat();
		setTraining(training);

		this.gradients = new double[this.currentFlatNetwork.getWeights().length];
		this.lastGradient = new double[this.currentFlatNetwork.getWeights().length];

		this.indexable = training;
		this.numThreads = 0;
		this.reportedException = null;
		this.shouldFixFlatSpot = true;
	}
	
	/**
	 * Change the dropout rate
	 * @param rate
	 * 			The dropout rate.
	 */
	public void setDroupoutRate(double rate) {
		this.dropoutRate = rate;
	}
	
	/**
	 * @return the current dropout rate
	 * 
	 */
	public double getDropoutRate() {
		return this.dropoutRate;
	}
	/**
	 * Should be called after training has completed and the iteration method
	 * will not be called any further.
	 */
	@Override
	public void finishTraining() {
		finishTraining(this.dropoutRate);
	}
	public void finishTraining(double dropoutRate) {
		if(!this.finalized) {
			final double[] weights = this.currentFlatNetwork.getWeights();
			if(dropoutRate > 0)
			{
				for (int i = 0; i < weights.length; i++) {
					weights[i] *= (1 - dropoutRate);
				}
			}
			this.finalized = true;
		}
		super.finishTraining();
	}

	/**
	 * @return the currentFlatNetwork
	 */
	public FlatNetwork getCurrentFlatNetwork() {
		return this.currentFlatNetwork;
	}

	/**
	 * {@inheritDoc}
	 */
	public MLMethod getMethod() {
		return this.network;
	}

	/**
	 * Perform one training iteration.
	 */
	public void iteration() {
		iteration(1);
	}

	/**
	 * Increase the iteration by one.
	 */
	public void rollIteration() {
		this.iteration++;
	}

	/**
	 * Process as pure batch (size 0). Batch size equal to training set size.
	 */
	private void processPureBatch() {
		calculateGradients();

		if (this.currentFlatNetwork.isLimited()) {
			learnLimited();
		} else {
			learn();
		}
	}

	private void processBatches() {
		if (this.workers == null) {
			init();
		}

		if (this.currentFlatNetwork.getHasContext()) {
			this.workers[0].getNetwork().clearContext();
		}

		this.workers[0].getErrorCalculation().reset();

		int lastLearn = 0;

		for (int i = 0; i < this.getTraining().size(); i++) {
			this.workers[0].run(i);

			lastLearn++;

			if (lastLearn++ >= this.batchSize) {
				if (this.currentFlatNetwork.isLimited()) {
					learnLimited();
				} else {
					learn();
					lastLearn = 0;
				}
			}
		}
		
		// handle any remaining learning
		if( lastLearn>0 ) {
			learn();
		}

		this.setError(this.workers[0].getErrorCalculation().calculate());

	}

	/**
	 * Perform the specified number of training iterations. This can be more
	 * efficient than single training iterations. This is particularly true if
	 * you are training with a GPU.
	 * 
	 * @param count
	 *            The number of training iterations.
	 */
	@Override
	public void iteration(final int count) {

		try {
			for (int i = 0; i < count; i++) {

				preIteration();

				rollIteration();

				if (this.batchSize == 0) {
					processPureBatch();
				} else {
					processBatches();
				}

				for (final GradientWorker worker : this.workers) {
					EngineArray.arrayCopy(this.currentFlatNetwork.getWeights(),
							0, worker.getWeights(), 0,
							this.currentFlatNetwork.getWeights().length);
				}

				if (this.currentFlatNetwork.getHasContext()) {
					copyContexts();
				}

				if (this.reportedException != null) {
					throw (new EncogError(this.reportedException));
				}

				postIteration();

				EncogLogging.log(EncogLogging.LEVEL_INFO,
						"Training iteration done, error: " + getError());

			}
		} catch (final ArrayIndexOutOfBoundsException ex) {
			EncogValidate.validateNetworkForTraining(this.network,
					getTraining());
			throw new EncogError(ex);
		}
	}

	/**
	 * Set the number of threads. Specify zero to tell Encog to automatically
	 * determine the best number of threads for the processor. If OpenCL is used
	 * as the target device, then this value is not used.
	 * 
	 * @param numThreads
	 *            The number of threads.
	 */
	public void setThreadCount(final int numThreads) {
		this.numThreads = numThreads;
	}

	@Override
	public int getThreadCount() {
		return this.numThreads;
	}

	/**
	 * Default is true. Call this with false to disable flat spot fix.
	 * 
	 * For more info on flat spot:
	 * 
	 * http://www.heatonresearch.com/wiki/Flat_Spot
	 * 
	 * @param b
	 *            True to fix flat spots, false otherwise.
	 */
	public void fixFlatSpot(boolean b) {
		this.shouldFixFlatSpot = b;
	}

	public void setErrorFunction(ErrorFunction ef) {
		this.ef = ef;
	}

	/**
	 * Calculate the gradients.
	 */
	public void calculateGradients() {
		if (this.workers == null) {
			init();
		}

		if (this.currentFlatNetwork.getHasContext()) {
			this.workers[0].getNetwork().clearContext();
		}

		this.totalError = 0;

		if (this.workers.length > 1) {

			final TaskGroup group = EngineConcurrency.getInstance()
					.createTaskGroup();

			for (final GradientWorker worker : this.workers) {
				EngineConcurrency.getInstance().processTask(worker, group);
			}

			group.waitForComplete();
		} else {
			this.workers[0].run();
		}

		this.setError(this.totalError / this.workers.length);

	}

	/**
	 * Copy the contexts to keep them consistent with multithreaded training.
	 */
	private void copyContexts() {

		// copy the contexts(layer outputO from each group to the next group
		for (int i = 0; i < (this.workers.length - 1); i++) {
			final double[] src = this.workers[i].getNetwork().getLayerOutput();
			final double[] dst = this.workers[i + 1].getNetwork()
					.getLayerOutput();
			EngineArray.arrayCopy(src, dst);
		}

		// copy the contexts from the final group to the real network
		EngineArray.arrayCopy(this.workers[this.workers.length - 1]
				.getNetwork().getLayerOutput(), this.currentFlatNetwork
				.getLayerOutput());
	}

	/**
	 * Init the process.
	 */
	private void init() {

		// fix flat spot, if needed
		this.flatSpot = new double[this.currentFlatNetwork
				.getActivationFunctions().length];

		if (this.shouldFixFlatSpot) {
			for (int i = 0; i < this.currentFlatNetwork
					.getActivationFunctions().length; i++) {
				final ActivationFunction af = this.currentFlatNetwork
						.getActivationFunctions()[i];

				if (af instanceof ActivationSigmoid) {
					this.flatSpot[i] = 0.1;
				} else {
					this.flatSpot[i] = 0.0;
				}
			}
		} else {
			EngineArray.fill(this.flatSpot, 0.0);
		}

		// setup workers

		// Do not use multi-threading for non-pure batch training.
		//
		// At some point it would be good to add multi-threading
		// for batch-sizes that are large enough.
		//
		// Multi-threading cannot be added for pure (size 1)
		// online training.
		if (this.batchSize != 0) {
			this.numThreads = 1;
		}

		final DetermineWorkload determine = new DetermineWorkload(
				this.numThreads, (int) this.indexable.getRecordCount());

		int actualThreadCount = determine.getThreadCount();

		this.workers = new GradientWorker[actualThreadCount];

		int index = 0;

		for (final IntRange r : determine.calculateWorkers()) {
			this.workers[index++] = new GradientWorker(
					this.currentFlatNetwork.clone(), this,
					this.indexable.openAdditional(), r.getLow(), r.getHigh(),
					this.flatSpot, this.ef);
		}

		initOthers();
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public void report(final double[] gradients, final double error,
			final Throwable ex) {
		synchronized (this) {
			if (ex == null) {

				for (int i = 0; i < gradients.length; i++) {
					this.gradients[i] += gradients[i];
				}
				this.totalError += error;
			} else {
				this.reportedException = ex;
			}
		}
	}

	/**
	 * Apply and learn.
	 */
	protected void learn() {
		final double[] weights = this.currentFlatNetwork.getWeights();
		if(this.dropoutRate > 0) {
			for (int i = 0; i < this.gradients.length; i++) {
				weights[i] += updateWeight(this.gradients, this.lastGradient, i, this.dropoutRate);
				this.gradients[i] = 0;
			}			
		} else {
			for (int i = 0; i < this.gradients.length; i++) {
				weights[i] += updateWeight(this.gradients, this.lastGradient, i);
				this.gradients[i] = 0;
			}			
		}
	}

	/**
	 * Apply and learn. This is the same as learn, but it checks to see if any
	 * of the weights are below the limit threshold. In this case, these weights
	 * are zeroed out. Having two methods allows the regular learn method, which
	 * is what is usually use, to be as fast as possible.
	 */
	protected void learnLimited() {
		final double limit = this.currentFlatNetwork.getConnectionLimit();
		final double[] weights = this.currentFlatNetwork.getWeights();
		if(this.dropoutRate > 0) {
			for (int i = 0; i < this.gradients.length; i++) {
				if (Math.abs(weights[i]) < limit) {
					weights[i] = 0;
				} else {
					weights[i] += updateWeight(this.gradients, this.lastGradient, i, this.dropoutRate);
				}
				this.gradients[i] = 0;
			}			
		} else {
			for (int i = 0; i < this.gradients.length; i++) {
				if (Math.abs(weights[i]) < limit) {
					weights[i] = 0;
				} else {
					weights[i] += updateWeight(this.gradients, this.lastGradient, i);
				}
				this.gradients[i] = 0;
			}			
		}
		for (int i = 0; i < this.gradients.length; i++) {
		}
	}

	public abstract void initOthers();

	/**
	 * Update a weight, the means by which weights are updated vary depending on
	 * the training.
	 * 
	 * @param gradients
	 *            The gradients.
	 * @param lastGradient
	 *            The last gradients.
	 * @param index
	 *            The index.
	 * @return The update value.
	 */
	public abstract double updateWeight(double[] gradients,
			double[] lastGradient, int index);

	/**
	 * Update a weight using dropout, the means by which weights are updated vary depending on
	 * the training.
	 * 
	 * @param gradients
	 *            The gradients.
	 * @param lastGradient
	 *            The last gradients.
	 * @param index
	 *            The index.
	 * @param dropoutRate
	 * 			  The dropout rate
	 * @return The update value.
	 */
	public abstract double updateWeight(double[] gradients,
			double[] lastGradient, int index, double dropoutRate);

	/**
	 * @return the lastGradient
	 */
	public double[] getLastGradient() {
		return lastGradient;
	}

	/**
	 * {@inheritDoc}
	 */
	public int getBatchSize() {
		return this.batchSize;
	}

	/**
	 * {@inheritDoc}
	 */
	public void setBatchSize(int theBatchSize) {
		this.batchSize = theBatchSize;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public double getL1() {
		return l1;
	}

	/**
	 * @param l1 How much to apply l1 regularization penalty, 0 (default) for none.
	 */
	public void setL1(double l1) {
		this.l1 = l1;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public double getL2() {
		return l2;
	}

	/**
	 * @param l2 How much to apply l2 regularization penalty, 0 (default) for none.
	 */
	public void setL2(double l2) {
		this.l2 = l2;
	}
	
	
	
}