update

Tinn.c

@@ -4,110 +4,114 @@
 #include <math.h>
 #include <time.h>
 
-static double error(Tinn t, double* tg)
+// Error function.
+static double err(double a, double b)
 {
-    double error = 0.0;
+    return 0.5 * pow(a - b, 2.0);
-    int i;
-    for(i = 0; i < t.nops; i++)
-        error += 0.5 * pow(tg[i] - t.o[i], 2.0);
-    return error;
 }
 
-static void backwards(Tinn t, double* in, double* tg, double rate)
+// Partial derivative of error function.
+static double pderr(double a, double b)
 {
-    double* x = t.w + t.nhid * t.nips;
+    return a - b;
-    int i;
-    for(i = 0; i < t.nhid; i++)
-    {
-        double sum = 0.0;
-        int j;
-        /* Calculate total error change with respect to output */
-        for(j = 0; j < t.nops; j++)
-        {
-            double a = t.o[j] - tg[j];
-            double b = t.o[j] * (1 - t.o[j]);
-            double c = x[j * t.nhid + i];
-            sum += a * b * c;
-        }
-        /* Correct weights in input to hidden layer */
-        for(j = 0; j < t.nips; j++)
-        {
-            double a = sum;
-            double b = t.h[i] * (1 - t.h[i]);
-            double c = in[j];
-            t.w[i * t.nips + j] -= rate * a * b * c;
-        }
-        /* Correct weights in hidden to output layer */
-        for(j = 0; j < t.nops; j++)
-        {
-            double a = t.o[j] - tg[j];
-            double b = t.o[j] * (1 - t.o[j]);
-            double c = t.h[i];
-            x[j * t.nhid + i] -= rate * a * b * c;
-        }
-    }
 }
 
-static double act(double net)
+// Total error.
+static double terr(const double* tg, const double* o, int size)
 {
-    return 1.0 / (1.0 + exp(-net));
+    double sum = 0.0;
+    for(int i = 0; i < size; i++)
+        sum += err(tg[i], o[i]);
+    return sum;
 }
 
-static double frand(void)
+// Activation function.
+static double act(double a)
+{
+    return 1.0 / (1.0 + exp(-a));
+}
+
+// Partial derivative of activation function.
+static double pdact(double a)
+{
+    return a * (1.0 - a);
+}
+
+// Floating point random from 0.0 - 1.0.
+static double frand()
 {
     return rand() / (double) RAND_MAX;
 }
 
-static void forewards(Tinn t, double* in)
+// Back propagation.
+static void backwards(const Tinn t, const double* in, const double* tg, double rate)
 {
     double* x = t.w + t.nhid * t.nips;
-    int i;
+    for(int i = 0; i < t.nhid; i++)
-    /* Calculate hidden layer neuron values */
-    for(i = 0; i < t.nhid; i++)
     {
         double sum = 0.0;
-        int j;
+        // Calculate total error change with respect to output.
-        for(j = 0; j < t.nips; j++)
+        for(int j = 0; j < t.nops; j++)
         {
-            double a = in[j];
+            double a = pderr(t.o[j], tg[j]);
-            double b = t.w[i * t.nips + j];
+            double b = pdact(t.o[j]);
-            sum += a * b;
+            sum += a * b * x[j * t.nhid + i];
+            // Correct weights in hidden to output layer.
+            x[j * t.nhid + i] -= rate * a * b * t.h[i];
         }
+        // Correct weights in input to hidden layer.
+        for(int j = 0; j < t.nips; j++)
+            t.w[i * t.nips + j] -= rate * sum * pdact(t.h[i]) * in[j];
+    }
+}
+
+// Forward propagation.
+static void forewards(const Tinn t, const double* in)
+{
+    double* x = t.w + t.nhid * t.nips;
+    // Calculate hidden layer neuron values.
+    for(int i = 0; i < t.nhid; i++)
+    {
+        double sum = 0.0;
+        for(int j = 0; j < t.nips; j++)
+            sum += in[j] * t.w[i * t.nips + j];
         t.h[i] = act(sum + t.b[0]);
     }
-    /* Calculate output layer neuron values */
+    // Calculate output layer neuron values.
-    for(i = 0; i < t.nops; i++)
+    for(int i = 0; i < t.nops; i++)
     {
         double sum = 0.0;
-        int j;
+        for(int j = 0; j < t.nhid; j++)
-        for(j = 0; j < t.nhid; j++)
+            sum += t.h[j] * x[i * t.nhid + j];
-        {
-            double a = t.h[j];
-            double b = x[i * t.nhid + j];
-            sum += a * b;
-        }
         t.o[i] = act(sum + t.b[1]);
     }
 }
 
-static void twrand(Tinn t)
+// Randomizes weights and biases.
+static void twrand(const Tinn t)
 {
     int wgts = t.nhid * (t.nips + t.nops);
-    int i;
+    for(int i = 0; i < wgts; i++) t.w[i] = frand() - 0.5;
-    for(i = 0; i < wgts; i++) t.w[i] = frand();
+    for(int i = 0; i < t.nb; i++) t.b[i] = frand() - 0.5;
-    for(i = 0; i < t.nb; i++) t.b[i] = frand();
 }
 
-double xttrain(Tinn t, double* in, double* tg, double rate)
+double* xpredict(const Tinn t, const double* in)
+{
+    forewards(t, in);
+    return t.o;
+}
+
+double xttrain(const Tinn t, const double* in, const double* tg, double rate)
 {
     forewards(t, in);
     backwards(t, in, tg, rate);
-    return error(t, tg);
+    return terr(tg, t.o, t.nops);
 }
 
 Tinn xtbuild(int nips, int nhid, int nops)
 {
     Tinn t;
+    // Tinn only supports one hidden layer so there are two biases.
     t.nb = 2;
     t.w = (double*) calloc(nhid * (nips + nops), sizeof(*t.w));
     t.b = (double*) calloc(t.nb, sizeof(*t.b));
@@ -121,7 +125,7 @@ Tinn xtbuild(int nips, int nhid, int nops)
     return t;
 }
 
-void xtfree(Tinn t)
+void xtfree(const Tinn t)
 {
     free(t.w);
     free(t.h);

Tinn.h

@@ -1,23 +1,25 @@
-#ifndef _TINN_H_
+#pragma once
-#define _TINN_H_
 
 typedef struct
 {
-    double* w;
+    double* w; // Weights.
-    double* b;
+    double* b; // Biases.
-    double* h;
+    double* h; // Hidden layer.
-    double* o;
+    double* o; // Output layer.
+
+    // Number of biases - always two - Tinn only supports a single hidden layer.
     int nb;
-    int nips;
+
-    int nhid;
+    int nips; // Number of inputs.
-    int nops;
+    int nhid; // Number of hidden neurons.
+    int nops; // Number of outputs.
 }
 Tinn;
 
-extern double xttrain(Tinn, double* in, double* tg, double rate);
+double xttrain(const Tinn, const double* in, const double* tg, double rate);
 
-extern Tinn xtbuild(int nips, int nhid, int nops);
+Tinn xtbuild(int nips, int nhid, int nops);
 
-extern void xtfree(Tinn);
+void xtfree(Tinn);
 
-#endif
+double* xpredict(const Tinn, const double* in);

test.c

@@ -1,20 +1,14 @@
 #include "Tinn.h"
-
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
-#include <time.h>
-
-#define toss(t, n) ((t*) malloc((n) * sizeof(t)))
-
-#define retoss(ptr, t, n) (ptr = (t*) realloc((ptr), (n) * sizeof(t)))
 
 typedef struct
 {
-    double** id;
+    double** in;
-    double** od;
+    double** tg;
-    int icols;
+    int nips;
-    int ocols;
+    int nops;
     int rows;
 }
 Data;
@@ -41,12 +35,12 @@ static char* readln(FILE* const file)
     int ch = EOF;
     int reads = 0;
     int size = 128;
-    char* line = toss(char, size);
+    char* line = ((char*) malloc((size) * sizeof(char)));
     while((ch = getc(file)) != '\n' && ch != EOF)
     {
         line[reads++] = ch;
         if(reads + 1 == size)
-            retoss(line, char, size *= 2);
+            line = (char*) realloc((line), (size *= 2) * sizeof(char));
     }
     line[reads] = '\0';
     return line;
@@ -54,30 +48,30 @@ static char* readln(FILE* const file)
 
 static double** new2d(const int rows, const int cols)
 {
-    double** row = toss(double*, rows);
+    double** row = (double**) malloc((rows) * sizeof(double*));
     for(int r = 0; r < rows; r++)
-        row[r] = toss(double, cols);
+        row[r] = (double*) malloc((cols) * sizeof(double));
     return row;
 }
 
-static Data ndata(const int icols, const int ocols, const int rows)
+static Data ndata(const int nips, const int nops, const int rows)
 {
     const Data data = {
-        new2d(rows, icols), new2d(rows, ocols), icols, ocols, rows
+        new2d(rows, nips), new2d(rows, nops), nips, nops, rows
     };
     return data;
 }
 
 static void parse(const Data data, char* line, const int row)
 {
-    const int cols = data.icols + data.ocols;
+    const int cols = data.nips + data.nops;
     for(int col = 0; col < cols; col++)
     {
-        const float val = atof(strtok(col == 0 ? line : NULL, " "));
+        const double val = atof(strtok(col == 0 ? line : NULL, " "));
-        if(col < data.icols)
+        if(col < data.nips)
-            data.id[row][col] = val;
+            data.in[row][col] = val;
         else
-            data.od[row][col - data.icols] = val;
+            data.tg[row][col - data.nips] = val;
     }
 }
 
@@ -85,11 +79,11 @@ static void dfree(const Data d)
 {
     for(int row = 0; row < d.rows; row++)
     {
-        free(d.id[row]);
+        free(d.in[row]);
-        free(d.od[row]);
+        free(d.tg[row]);
     }
-    free(d.id);
+    free(d.in);
-    free(d.od);
+    free(d.tg);
 }
 
 static void shuffle(const Data d)
@@ -97,28 +91,29 @@ static void shuffle(const Data d)
     for(int a = 0; a < d.rows; a++)
     {
         const int b = rand() % d.rows;
-        double* ot = d.od[a];
+        double* ot = d.tg[a];
-        double* it = d.id[a];
+        double* it = d.in[a];
         // Swap output.
-        d.od[a] = d.od[b];
+        d.tg[a] = d.tg[b];
-        d.od[b] = ot;
+        d.tg[b] = ot;
         // Swap input.
-        d.id[a] = d.id[b];
+        d.in[a] = d.in[b];
-        d.id[b] = it;
+        d.in[b] = it;
     }
 }
 
-static Data build(const char* path, const int icols, const int ocols)
+static Data build(const char* path, const int nips, const int nops)
 {
     FILE* file = fopen(path, "r");
     if(file == NULL)
     {
         printf("Could not open %s\n", path);
-        printf("Get the training data: \n");
+        printf("Get it from the machine learning database: ");
+        printf("wget http://archive.ics.uci.edu/ml/machine-learning-databases/semeion/semeion.data\n");
         exit(1);
     }
     const int rows = lns(file);
-    Data data = ndata(icols, ocols, rows);
+    Data data = ndata(nips, nops, rows);
     for(int row = 0; row < rows; row++)
     {
         char* line = readln(file);
@@ -129,22 +124,40 @@ static Data build(const char* path, const int icols, const int ocols)
     return data;
 }
 
-int main(void)
+int main()
 {
-    const Data data = build("semeion.data", 256, 10);
+    // Input and output size is harded coded here,
-    shuffle(data);
+    // so make sure the training data sizes match.
-    const Tinn tinn = xtbuild(data.icols, 64, data.ocols);
+    const int nips = 256;
-    for(int i = 0; i < 10000; i++)
+    const int nops = 10;
+    // Hyper Parameters.
+    // Learning rate is annealed and thus not constant.
+    const int nhid = 32;
+    double rate = 0.5;
+    // Load the training set.
+    const Data data = build("semeion.data", nips, nops);
+    // Rock and roll.
+    const Tinn tinn = xtbuild(nips, nhid, nops);
+    for(int i = 0; i < 100; i++)
     {
+        shuffle(data);
         double error = 0.0;
         for(int j = 0; j < data.rows; j++)
         {
-            double* in = data.id[j];
+            const double* const in = data.in[j];
-            double* tg = data.od[j];
+            const double* const tg = data.tg[j];
-            //error += xttrain(tinn, in, tg, 0.5);
+            error += xttrain(tinn, in, tg, rate);
         }
-        printf("%.12f\n", error);
+        printf("error %.12f :: rate %f\n", error / data.rows, rate);
+        rate *= 0.99;
     }
+    // Ideally, you would load a testing set for predictions,
+    // but for the sake of brevity the training set is reused.
+    const double* const in = data.in[0];
+    const double* const tg = data.tg[0];
+    const double* const pd = xpredict(tinn, in);
+    for(int i = 0; i < data.nops; i++) { printf("%f ", tg[i]); } printf("\n");
+    for(int i = 0; i < data.nops; i++) { printf("%f ", pd[i]); } printf("\n");
     xtfree(tinn);
     dfree(data);
     return 0;