Seperated backpropagation algorithm from neural network. Added supervised trainer abstraction.

There's no logical reason to have the backpropagation algorithm to reside within the neural network; you should be able to use different learning algorithms on the same neural network.
I believe first-year-me originally intended to use inheritance to override the backpropgation in NeuralNetwork subclasses. Bad idea.

Author: PatrickShaw

GeNeural/GeNeural.csproj

@@ -36,6 +36,7 @@
   <ItemGroup>
     <Compile Include="Activations\IActivator.cs" />
     <Compile Include="Activations\LogisticActivator.cs" />
+    <Compile Include="Training\Backpropagation\StandardBackpropagationTrainer.cs" />
     <Compile Include="GeneticNeuralNetworkFacilitator.cs" />
     <Compile Include="Genetic\DisimilarityFunctions.cs" />
     <Compile Include="Genetic\EfficiencyErrorFunction.cs" />
@@ -49,12 +50,14 @@
     <Compile Include="IDeepCloneable.cs" />
     <Compile Include="IGenetic.cs" />
     <Compile Include="IMutatable.cs" />
+    <Compile Include="INeuralNetwork.cs" />
+    <Compile Include="Training\ISupervisedTrainer.cs" />
     <Compile Include="NeuralNetwork.cs" />
     <Compile Include="Helpers.cs" />
     <Compile Include="Neuron.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Sorter.cs" />
-    <Compile Include="TopologyTrainer.cs" />
+    <Compile Include="Training\Topology\TopologyTrainer.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildExtensionsPath32)\Microsoft\Portable\$(TargetFrameworkVersion)\Microsoft.Portable.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 

GeNeural/GeneticNeuralNetworkFacilitator.cs

@@ -18,7 +18,7 @@ public class GeneticNeuralNetworkFacilitator : IMutatable, IDeepCloneable<Geneti
 
         private double neuronMutationFactorVarianceFactor = 0.01;
         private double neuronMutationFactor = 0.50; // Adds round(-x to x) neurons per layer
-        NeuralNetwork network;
+        private NeuralNetwork network;
         public GeneticNeuralNetworkFacilitator(NeuralNetwork network)
         {
             this.network = network;

GeNeural/INeuralNetwork.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace GeNeural
+{
+    /// TODO: There is likley a better name for this interface. Not all neural networks 
+    /// naturally uphold this interface and many other machine learning algorithms do.
+    /// <summary>
+    /// 
+    /// </summary>
+    interface INeuralNetwork
+    {
+        double[] CalculateOutputs(double[] inputs);
+    }
+}

GeNeural/NeuralNetwork.cs

@@ -13,7 +13,7 @@ namespace GeNeural
     using System.Text;
     using System.Threading.Tasks;
     public class NotEnoughLayersException : Exception { }
-    public class NeuralNetwork : IDeepCloneable<NeuralNetwork>
+    public class NeuralNetwork : IDeepCloneable<NeuralNetwork>, INeuralNetwork
     {
         private const int INPUT_NEURON_WEIGHTS_COUNT = 2;
         private Neuron[][] neurons;
@@ -97,7 +97,7 @@ public double[] CalculateOutputs(double[] inputs)
             double[][] outputs = CalculateAllOutputs(inputs);
             return outputs[outputs.Length - 1];
         }
-        private double[][] CalculateAllOutputs(double[] inputs)
+        public double[][] CalculateAllOutputs(double[] inputs)
         {
             double[][] outputs = new double[neurons.Length][];
             outputs[0] = new double[neurons[0].Length];
@@ -204,51 +204,6 @@ public void ResetMomentum()
                 foreach (Neuron neuron in layer)
                     neuron.ResetMomentum();
         }
-        public void BackPropagate(double[] inputs, double[] desiredOutputs)
-        {
-            double[][] outputs = CalculateAllOutputs(inputs);
-            double[][] weirdDThing = new double[outputs.Length][];
-            for (int l = 0; l < outputs.Length; l++)
-                weirdDThing[l] = new double[outputs[l].Length];
-            for (int n = 0; n < outputs[outputs.Length - 1].Length; n++)
-            {
-                double neuronOutput = outputs[outputs.Length - 1][n];
-                weirdDThing[weirdDThing.Length - 1][n] = (neuronOutput - desiredOutputs[n]) * neuronOutput * (1 - neuronOutput);
-            }
-            for (int l = outputs.Length - 2; l >= 0; l--)
-            {
-                for (int n = 0; n < neurons[l].Length; n++)
-                {
-                    double neuronOutput = outputs[l][n];
-                    double sumThing = 0;
-                    for (int n2 = 0; n2 < neurons[l + 1].Length; n2++)
-                    {
-                        sumThing += weirdDThing[l + 1][n2] * neurons[l + 1][n2].Weights[n];
-                    }
-                    weirdDThing[l][n] = sumThing * neuronOutput * (1 - neuronOutput);
-                }
-            }
-            const double learningFactor = 0.1;
-            for (int n = 0; n < neurons[0].Length; n++)
-            {
-                neurons[0][n].Weights[0] -= learningFactor * weirdDThing[0][n] * -1;
-                for (int n2 = 0; n2 < inputs.Length; n2++)
-                {
-                      neurons[0][n].Weights[n2 + 1] -= learningFactor * weirdDThing[0][n] * inputs[n2];
-                }
-            }
-            for (int l = 1; l < neurons.Length; l++)
-            {
-                for (int n = 0; n < neurons[l].Length; n++)
-                {
-                    neurons[l][n].Weights[0] -= learningFactor * weirdDThing[l][n] * -1;
-                    for (int n2 = 0; n2 < neurons[l - 1].Length; n2++)
-                    {
-                        neurons[l][n].Weights[n2 + 1] -= learningFactor * weirdDThing[l][n] * outputs[l - 1][n2];
-                    }
-                }
-            }
-        }
         public void AddOutputNeuron(Neuron neuron)
         {
             int layerIndex = neurons.Length - 1;

GeNeural/Sorter.cs

@@ -6,7 +6,7 @@
 
 namespace GeNeural
 {
-    static class Sorter
+    public static class Sorter
     {
         private static void Swap<T>(T[] array, int i1, int i2)
         {

GeNeural/Training/Backpropagation/StandardBackpropagationTrainer.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace GeNeural.Training.Backpropagation
+{
+    public sealed class StandardBackpropagationTrainer : ISupervisedTrainer<NeuralNetwork>
+    {
+        public void Backpropagation(NeuralNetwork neuralNetwork, double[] inputs, double[] desiredOutputs)
+        {
+            // We need to calculate the current outputs, given a set of inputs in order to do backpropagation.
+            double[][] outputs = neuralNetwork.CalculateAllOutputs(inputs);
+            // TODO: Revisit 'weirdDThing'.
+            double[][] weirdDThing = new double[outputs.Length][];
+            for (int l = 0; l < outputs.Length; l++)
+            {
+                weirdDThing[l] = new double[outputs[l].Length];
+            }
+            for (int n = 0; n < outputs[outputs.Length - 1].Length; n++)
+            {
+                double neuronOutput = outputs[outputs.Length - 1][n];
+                weirdDThing[weirdDThing.Length - 1][n] = (neuronOutput - desiredOutputs[n]) * neuronOutput * (1 - neuronOutput);
+            }
+            for (int l = outputs.Length - 2; l >= 0; l--)
+            {
+                int inputLength = neuralNetwork.GetLayer(l).Length;
+                Neuron[] currentLayer = neuralNetwork.GetLayer(l + 1);
+                for (int n = 0; n < inputLength; n++)
+                {
+                    double neuronOutput = outputs[l][n];
+                    double sumThing = 0;
+                    for (int n2 = 0; n2 < currentLayer.Length; n2++)
+                    {
+                        sumThing += weirdDThing[l + 1][n2] * currentLayer[n2].Weights[n];
+                    }
+                    weirdDThing[l][n] = sumThing * neuronOutput * (1 - neuronOutput);
+                }
+            }
+            const double learningFactor = 0.1;
+            // Now we actually modify the the weights of the neurons.
+            // The first layer is a special case it doesn't have any previous layers to deal with.
+            Neuron[] firstLayer = neuralNetwork.GetLayer(0);
+            for (int n = 0; n < firstLayer.Length; n++)
+            {
+                firstLayer[n].Weights[0] -= learningFactor * weirdDThing[0][n] * -1;
+                for (int n2 = 0; n2 < inputs.Length; n2++)
+                {
+                    firstLayer[n].Weights[n2 + 1] -= learningFactor * weirdDThing[0][n] * inputs[n2];
+                }
+            }
+            // Now modify the weights for the other neural networks.
+            for (int l = 1; l < neuralNetwork.LayerCount; l++)
+            {
+                Neuron[] currentLayer = neuralNetwork.GetLayer(l);
+                Neuron[] previousLayer = neuralNetwork.GetLayer(l - 1);
+                for (int n = 0; n < currentLayer.Length; n++)
+                {
+                    currentLayer[n].Weights[0] -= learningFactor * weirdDThing[l][n] * -1;
+                    for (int n2 = 0; n2 < previousLayer.Length; n2++)
+                    {
+                        currentLayer[n].Weights[n2 + 1] -= learningFactor * weirdDThing[l][n] * outputs[l - 1][n2];
+                    }
+                }
+            }
+        }
+
+        public void Train(NeuralNetwork trainable, double[] trainingInputs, double[] trainingOutputs)
+        {
+            Backpropagation(trainable, trainingInputs, trainingOutputs);
+        }
+    }
+}

GeNeural/Training/ISupervisedTrainer.cs

@@ -0,0 +1,13 @@
+namespace GeNeural.Training
+{
+    public interface ISupervisedTrainer<T>
+    {
+        /// <summary>
+        /// Trains a given trainable entity to output a certain set of outputs, given a certain set of inputs.
+        /// </summary>
+        /// <param name="trainable"></param>
+        /// <param name="trainingInputs"></param>
+        /// <param name="trainingOutputs"></param>
+        void Train(T trainable, double[] trainingInputs, double[] trainingOutputs);
+    }
+}

GeNeural/TopologyTrainer.cs renamed to GeNeural/Training/Topology/TopologyTrainer.cs

@@ -4,30 +4,29 @@
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-
-namespace GeNeural.Trainers
+namespace GeNeural.Training.Topology
 {
-    public class TopologyTrainer
+    public sealed class TopologyTrainer
     {
-        public static NeuralNetwork ConfigureTopology(double[][] trainingInputs, double [][] trainingDesiredOutputs, double[][] testInputs, double[][] testDesiredOutputs)
+        public static NeuralNetwork ConfigureTopology(ISupervisedTrainer<NeuralNetwork> trainer, double[][] trainingInputs, double [][] trainingDesiredOutputs, double[][] testInputs, double[][] testDesiredOutputs)
         {
             NeuralNetwork untrainedStubNetwork = new NeuralNetwork(trainingInputs[0].Length,new int[] { trainingInputs[0].Length, trainingDesiredOutputs[0].Length});
             NeuralNetwork trainedStubNetwork = untrainedStubNetwork.DeepClone();
-            Train(trainedStubNetwork, trainingInputs, trainingDesiredOutputs);
+            Train(trainer, trainedStubNetwork, trainingInputs, trainingDesiredOutputs);
             double stubNetworkError = GetTotalError(trainedStubNetwork, trainingInputs, trainingDesiredOutputs) + GetTotalError(trainedStubNetwork, testInputs, testDesiredOutputs);
             while(true)
             {
                 NeuralNetwork untrainedLayerNetwork = untrainedStubNetwork.DeepClone();
                 untrainedLayerNetwork.InsertAfterLayer(untrainedLayerNetwork.LayerCount - 1);
                 NeuralNetwork trainedLayerNetwork = untrainedStubNetwork.DeepClone();
-                Train(trainedLayerNetwork, trainingInputs, trainingDesiredOutputs);
+                Train(trainer, trainedLayerNetwork, trainingInputs, trainingDesiredOutputs);
                 double layerNetworkError = GetTotalError(trainedLayerNetwork, trainingInputs, trainingDesiredOutputs) + GetTotalError(trainedLayerNetwork, testInputs, testDesiredOutputs);
                 while(true)
                 {
                     NeuralNetwork untrainedNeuronNetwork = untrainedLayerNetwork.DeepClone();
                     untrainedNeuronNetwork.AddNeuronNonDestructive(untrainedNeuronNetwork.LayerCount - 2);
                     NeuralNetwork trainedNeuronNetwork = untrainedNeuronNetwork.DeepClone();
-                    Train(trainedNeuronNetwork, trainingInputs, trainingDesiredOutputs);
+                    Train(trainer, trainedNeuronNetwork, trainingInputs, trainingDesiredOutputs);
                     double neuronNetworkError = GetTotalError(trainedNeuronNetwork, trainingInputs, trainingDesiredOutputs);
                     if(neuronNetworkError < layerNetworkError)
                     {
@@ -69,7 +68,7 @@ public static double GetError(double desiredOutput, double actualOutput)
             double difference = desiredOutput - actualOutput;
             return difference * difference;
         }
-        public static void Train(NeuralNetwork network, double[][] inputs, double[][] desiredOutputs)
+        public static void Train<T>(ISupervisedTrainer<T> trainer, T network, double[][] inputs, double[][] desiredOutputs) where T : NeuralNetwork
         {
 
             for (int _ = 0; _ < 100; _++)
@@ -78,7 +77,7 @@ public static void Train(NeuralNetwork network, double[][] inputs, double[][] de
                 {
                     for (int i = 0; i < 50; i++)
                     {
-                        network.BackPropagate(inputs[t], desiredOutputs[t]);
+                        trainer.Train(network, inputs[t], desiredOutputs[t]);
                     }
                 }
             }

GeNeural/obj/Debug/GeNeural.csproj.FileListAbsolute.txt

@@ -7,3 +7,8 @@ C:\Users\eastd\Documents\Visual Studio 2015\Projects\GeNeural\GeNeural\bin\Debug
 C:\Users\eastd\Documents\Visual Studio 2015\Projects\GeNeural\GeNeural\obj\Debug\GeNeural.dll
 C:\Users\eastd\Documents\Visual Studio 2015\Projects\GeNeural\GeNeural\obj\Debug\GeNeural.pdb
 C:\Users\eastd\Documents\Visual Studio 2015\Projects\GeNeural\GeNeural\obj\Debug\GeNeural.csprojResolveAssemblyReference.cache
+C:\Users\eastd\Code\GeNeural\GeNeural\obj\Debug\GeNeural.csprojResolveAssemblyReference.cache
+C:\Users\eastd\Code\GeNeural\GeNeural\bin\Debug\GeNeural.dll
+C:\Users\eastd\Code\GeNeural\GeNeural\bin\Debug\GeNeural.pdb
+C:\Users\eastd\Code\GeNeural\GeNeural\obj\Debug\GeNeural.dll
+C:\Users\eastd\Code\GeNeural\GeNeural\obj\Debug\GeNeural.pdb