From 171bd195e91f2217e2a0454024d586e9b19934d6 Mon Sep 17 00:00:00 2001 From: Gustav Louw Date: Fri, 30 Mar 2018 13:04:37 -0700 Subject: [PATCH] update --- Tinn.c | 136 +++++++++++++++++++++++++++++---------------------------- Tinn.h | 28 ++++++------ test.c | 99 +++++++++++++++++++++++------------------ 3 files changed, 141 insertions(+), 122 deletions(-) diff --git a/Tinn.c b/Tinn.c index 8f8cfdf..fe0e9de 100644 --- a/Tinn.c +++ b/Tinn.c @@ -4,110 +4,114 @@ #include #include -static double error(Tinn t, double* tg) +// Error function. +static double err(double a, double b) { - double error = 0.0; - int i; - for(i = 0; i < t.nops; i++) - error += 0.5 * pow(tg[i] - t.o[i], 2.0); - return error; + return 0.5 * pow(a - b, 2.0); } -static void backwards(Tinn t, double* in, double* tg, double rate) +// Partial derivative of error function. +static double pderr(double a, double b) { - double* x = t.w + t.nhid * t.nips; - int i; - for(i = 0; i < t.nhid; i++) - { - double sum = 0.0; - int j; - /* Calculate total error change with respect to output */ - for(j = 0; j < t.nops; j++) - { - double a = t.o[j] - tg[j]; - double b = t.o[j] * (1 - t.o[j]); - double c = x[j * t.nhid + i]; - sum += a * b * c; - } - /* Correct weights in input to hidden layer */ - for(j = 0; j < t.nips; j++) - { - double a = sum; - double b = t.h[i] * (1 - t.h[i]); - double c = in[j]; - t.w[i * t.nips + j] -= rate * a * b * c; - } - /* Correct weights in hidden to output layer */ - for(j = 0; j < t.nops; j++) - { - double a = t.o[j] - tg[j]; - double b = t.o[j] * (1 - t.o[j]); - double c = t.h[i]; - x[j * t.nhid + i] -= rate * a * b * c; - } - } + return a - b; } -static double act(double net) +// Total error. +static double terr(const double* tg, const double* o, int size) { - return 1.0 / (1.0 + exp(-net)); + double sum = 0.0; + for(int i = 0; i < size; i++) + sum += err(tg[i], o[i]); + return sum; } -static double frand(void) +// Activation function. +static double act(double a) +{ + return 1.0 / (1.0 + exp(-a)); +} + +// Partial derivative of activation function. +static double pdact(double a) +{ + return a * (1.0 - a); +} + +// Floating point random from 0.0 - 1.0. +static double frand() { return rand() / (double) RAND_MAX; } -static void forewards(Tinn t, double* in) +// Back propagation. +static void backwards(const Tinn t, const double* in, const double* tg, double rate) { double* x = t.w + t.nhid * t.nips; - int i; - /* Calculate hidden layer neuron values */ - for(i = 0; i < t.nhid; i++) + for(int i = 0; i < t.nhid; i++) { double sum = 0.0; - int j; - for(j = 0; j < t.nips; j++) + // Calculate total error change with respect to output. + for(int j = 0; j < t.nops; j++) { - double a = in[j]; - double b = t.w[i * t.nips + j]; - sum += a * b; + double a = pderr(t.o[j], tg[j]); + double b = pdact(t.o[j]); + sum += a * b * x[j * t.nhid + i]; + // Correct weights in hidden to output layer. + x[j * t.nhid + i] -= rate * a * b * t.h[i]; } + // Correct weights in input to hidden layer. + for(int j = 0; j < t.nips; j++) + t.w[i * t.nips + j] -= rate * sum * pdact(t.h[i]) * in[j]; + } +} + +// Forward propagation. +static void forewards(const Tinn t, const double* in) +{ + double* x = t.w + t.nhid * t.nips; + // Calculate hidden layer neuron values. + for(int i = 0; i < t.nhid; i++) + { + double sum = 0.0; + for(int j = 0; j < t.nips; j++) + sum += in[j] * t.w[i * t.nips + j]; t.h[i] = act(sum + t.b[0]); } - /* Calculate output layer neuron values */ - for(i = 0; i < t.nops; i++) + // Calculate output layer neuron values. + for(int i = 0; i < t.nops; i++) { double sum = 0.0; - int j; - for(j = 0; j < t.nhid; j++) - { - double a = t.h[j]; - double b = x[i * t.nhid + j]; - sum += a * b; - } + for(int j = 0; j < t.nhid; j++) + sum += t.h[j] * x[i * t.nhid + j]; t.o[i] = act(sum + t.b[1]); } } -static void twrand(Tinn t) +// Randomizes weights and biases. +static void twrand(const Tinn t) { int wgts = t.nhid * (t.nips + t.nops); - int i; - for(i = 0; i < wgts; i++) t.w[i] = frand(); - for(i = 0; i < t.nb; i++) t.b[i] = frand(); + for(int i = 0; i < wgts; i++) t.w[i] = frand() - 0.5; + for(int i = 0; i < t.nb; i++) t.b[i] = frand() - 0.5; } -double xttrain(Tinn t, double* in, double* tg, double rate) +double* xpredict(const Tinn t, const double* in) +{ + forewards(t, in); + return t.o; +} + +double xttrain(const Tinn t, const double* in, const double* tg, double rate) { forewards(t, in); backwards(t, in, tg, rate); - return error(t, tg); + return terr(tg, t.o, t.nops); } Tinn xtbuild(int nips, int nhid, int nops) { Tinn t; + // Tinn only supports one hidden layer so there are two biases. t.nb = 2; t.w = (double*) calloc(nhid * (nips + nops), sizeof(*t.w)); t.b = (double*) calloc(t.nb, sizeof(*t.b)); @@ -121,7 +125,7 @@ Tinn xtbuild(int nips, int nhid, int nops) return t; } -void xtfree(Tinn t) +void xtfree(const Tinn t) { free(t.w); free(t.h); diff --git a/Tinn.h b/Tinn.h index c084331..baec2f6 100644 --- a/Tinn.h +++ b/Tinn.h @@ -1,23 +1,25 @@ -#ifndef _TINN_H_ -#define _TINN_H_ +#pragma once typedef struct { - double* w; - double* b; - double* h; - double* o; + double* w; // Weights. + double* b; // Biases. + double* h; // Hidden layer. + double* o; // Output layer. + + // Number of biases - always two - Tinn only supports a single hidden layer. int nb; - int nips; - int nhid; - int nops; + + int nips; // Number of inputs. + int nhid; // Number of hidden neurons. + int nops; // Number of outputs. } Tinn; -extern double xttrain(Tinn, double* in, double* tg, double rate); +double xttrain(const Tinn, const double* in, const double* tg, double rate); -extern Tinn xtbuild(int nips, int nhid, int nops); +Tinn xtbuild(int nips, int nhid, int nops); -extern void xtfree(Tinn); +void xtfree(Tinn); -#endif +double* xpredict(const Tinn, const double* in); diff --git a/test.c b/test.c index 6fcb16e..3423374 100644 --- a/test.c +++ b/test.c @@ -1,20 +1,14 @@ #include "Tinn.h" - #include #include #include -#include - -#define toss(t, n) ((t*) malloc((n) * sizeof(t))) - -#define retoss(ptr, t, n) (ptr = (t*) realloc((ptr), (n) * sizeof(t))) typedef struct { - double** id; - double** od; - int icols; - int ocols; + double** in; + double** tg; + int nips; + int nops; int rows; } Data; @@ -41,12 +35,12 @@ static char* readln(FILE* const file) int ch = EOF; int reads = 0; int size = 128; - char* line = toss(char, size); + char* line = ((char*) malloc((size) * sizeof(char))); while((ch = getc(file)) != '\n' && ch != EOF) { line[reads++] = ch; if(reads + 1 == size) - retoss(line, char, size *= 2); + line = (char*) realloc((line), (size *= 2) * sizeof(char)); } line[reads] = '\0'; return line; @@ -54,30 +48,30 @@ static char* readln(FILE* const file) static double** new2d(const int rows, const int cols) { - double** row = toss(double*, rows); + double** row = (double**) malloc((rows) * sizeof(double*)); for(int r = 0; r < rows; r++) - row[r] = toss(double, cols); + row[r] = (double*) malloc((cols) * sizeof(double)); return row; } -static Data ndata(const int icols, const int ocols, const int rows) +static Data ndata(const int nips, const int nops, const int rows) { const Data data = { - new2d(rows, icols), new2d(rows, ocols), icols, ocols, rows + new2d(rows, nips), new2d(rows, nops), nips, nops, rows }; return data; } static void parse(const Data data, char* line, const int row) { - const int cols = data.icols + data.ocols; + const int cols = data.nips + data.nops; for(int col = 0; col < cols; col++) { - const float val = atof(strtok(col == 0 ? line : NULL, " ")); - if(col < data.icols) - data.id[row][col] = val; + const double val = atof(strtok(col == 0 ? line : NULL, " ")); + if(col < data.nips) + data.in[row][col] = val; else - data.od[row][col - data.icols] = val; + data.tg[row][col - data.nips] = val; } } @@ -85,11 +79,11 @@ static void dfree(const Data d) { for(int row = 0; row < d.rows; row++) { - free(d.id[row]); - free(d.od[row]); + free(d.in[row]); + free(d.tg[row]); } - free(d.id); - free(d.od); + free(d.in); + free(d.tg); } static void shuffle(const Data d) @@ -97,28 +91,29 @@ static void shuffle(const Data d) for(int a = 0; a < d.rows; a++) { const int b = rand() % d.rows; - double* ot = d.od[a]; - double* it = d.id[a]; + double* ot = d.tg[a]; + double* it = d.in[a]; // Swap output. - d.od[a] = d.od[b]; - d.od[b] = ot; + d.tg[a] = d.tg[b]; + d.tg[b] = ot; // Swap input. - d.id[a] = d.id[b]; - d.id[b] = it; + d.in[a] = d.in[b]; + d.in[b] = it; } } -static Data build(const char* path, const int icols, const int ocols) +static Data build(const char* path, const int nips, const int nops) { FILE* file = fopen(path, "r"); if(file == NULL) { printf("Could not open %s\n", path); - printf("Get the training data: \n"); + printf("Get it from the machine learning database: "); + printf("wget http://archive.ics.uci.edu/ml/machine-learning-databases/semeion/semeion.data\n"); exit(1); } const int rows = lns(file); - Data data = ndata(icols, ocols, rows); + Data data = ndata(nips, nops, rows); for(int row = 0; row < rows; row++) { char* line = readln(file); @@ -129,22 +124,40 @@ static Data build(const char* path, const int icols, const int ocols) return data; } -int main(void) +int main() { - const Data data = build("semeion.data", 256, 10); - shuffle(data); - const Tinn tinn = xtbuild(data.icols, 64, data.ocols); - for(int i = 0; i < 10000; i++) + // Input and output size is harded coded here, + // so make sure the training data sizes match. + const int nips = 256; + const int nops = 10; + // Hyper Parameters. + // Learning rate is annealed and thus not constant. + const int nhid = 32; + double rate = 0.5; + // Load the training set. + const Data data = build("semeion.data", nips, nops); + // Rock and roll. + const Tinn tinn = xtbuild(nips, nhid, nops); + for(int i = 0; i < 100; i++) { + shuffle(data); double error = 0.0; for(int j = 0; j < data.rows; j++) { - double* in = data.id[j]; - double* tg = data.od[j]; - //error += xttrain(tinn, in, tg, 0.5); + const double* const in = data.in[j]; + const double* const tg = data.tg[j]; + error += xttrain(tinn, in, tg, rate); } - printf("%.12f\n", error); + printf("error %.12f :: rate %f\n", error / data.rows, rate); + rate *= 0.99; } + // Ideally, you would load a testing set for predictions, + // but for the sake of brevity the training set is reused. + const double* const in = data.in[0]; + const double* const tg = data.tg[0]; + const double* const pd = xpredict(tinn, in); + for(int i = 0; i < data.nops; i++) { printf("%f ", tg[i]); } printf("\n"); + for(int i = 0; i < data.nops; i++) { printf("%f ", pd[i]); } printf("\n"); xtfree(tinn); dfree(data); return 0;