Neural Networks Source Codes - The Backpropagation Network -------------------------------------------------------------------------------- The Backpropagation Network -------------------------------------------------------------------------------- This program is copyright (c) 1996 by the author. It is made available as is, and no warranty - about the program, its performance, or its conformity to any specification - is given or implied. It may be used, modified, and distributed freely for private and commercial purposes, as long as the original author is credited as part of the final work. Backpropagation Network Simulator /****************************************************************************** ==================================================== Network: Backpropagation Network with Bias Terms and Momentum ==================================================== Application: Time-Series Forecasting Prediction of the Annual Number of Sunspots Author: Karsten Kutza Date: 17.4.96 Reference: D.E. Rumelhart, G.E. Hinton, R.J. Williams Learning Internal Representations by Error Propagation in: D.E. Rumelhart, J.L. McClelland (Eds.) Parallel Distributed Processing, Volume 1 MIT Press, Cambridge, MA, pp. 318-362, 1986 ******************************************************************************/ /****************************************************************************** D E C L A R A T I O N S ******************************************************************************/ #include #include #include typedef int BOOL; typedef int INT; typedef double REAL; #define FALSE 0 #define TRUE 1 #define NOT ! #define AND && #define OR || #define MIN_REAL -HUGE_VAL #define MAX_REAL +HUGE_VAL #define MIN(x,y) ((x)<(y) ? (x) : (y)) #define MAX(x,y) ((x)>(y) ? (x) : (y)) #define LO 0.1 #define HI 0.9 #define BIAS 1 #define sqr(x) ((x)*(x)) typedef struct { /* A LAYER OF A NET: */ INT Units; /* - number of units in this layer */ REAL* Output; /* - output of ith unit */ REAL* Error; /* - error term of ith unit */ REAL** Weight; /* - connection weights to ith unit */ REAL** WeightSave; /* - saved weights for stopped training */ REAL** dWeight; /* - last weight deltas for momentum */ } LAYER; typedef struct { /* A NET: */ LAYER** Layer; /* - layers of this net */ LAYER* InputLayer; /* - input layer */ LAYER* OutputLayer; /* - output layer */ REAL Alpha; /* - momentum factor */ REAL Eta; /* - learning rate */ REAL Gain; /* - gain of sigmoid function */ REAL Error; /* - total net error */ } NET; /****************************************************************************** R A N D O M S D R A W N F R O M D I S T R I B U T I O N S ******************************************************************************/ void InitializeRandoms() { srand(4711); } INT RandomEqualINT(INT Low, INT High) { return rand() % (High-Low+1) + Low; } REAL RandomEqualREAL(REAL Low, REAL High) { return ((REAL) rand() / RAND_MAX) * (High-Low) + Low; } /****************************************************************************** A P P L I C A T I O N - S P E C I F I C C O D E ******************************************************************************/ #define NUM_LAYERS 3 #define N 30 #define M 1 INT Units[NUM_LAYERS] = {N, 10, M}; #define FIRST_YEAR 1700 #define NUM_YEARS 280 #define TRAIN_LWB (N) #define TRAIN_UPB (179) #define TRAIN_YEARS (TRAIN_UPB - TRAIN_LWB + 1) #define TEST_LWB (180) #define TEST_UPB (259) #define TEST_YEARS (TEST_UPB - TEST_LWB + 1) #define EVAL_LWB (260) #define EVAL_UPB (NUM_YEARS - 1) #define EVAL_YEARS (EVAL_UPB - EVAL_LWB + 1) REAL Sunspots_[NUM_YEARS]; REAL Sunspots [NUM_YEARS] = { 0.0262, 0.0575, 0.0837, 0.1203, 0.1883, 0.3033, 0.1517, 0.1046, 0.0523, 0.0418, 0.0157, 0.0000, 0.0000, 0.0105, 0.0575, 0.1412, 0.2458, 0.3295, 0.3138, 0.2040, 0.1464, 0.1360, 0.1151, 0.0575, 0.1098, 0.2092, 0.4079, 0.6381, 0.5387, 0.3818, 0.2458, 0.1831, 0.0575, 0.0262, 0.0837, 0.1778, 0.3661, 0.4236, 0.5805, 0.5282, 0.3818, 0.2092, 0.1046, 0.0837, 0.0262, 0.0575, 0.1151, 0.2092, 0.3138, 0.4231, 0.4362, 0.2495, 0.2500, 0.1606, 0.0638, 0.0502, 0.0534, 0.1700, 0.2489, 0.2824, 0.3290, 0.4493, 0.3201, 0.2359, 0.1904, 0.1093, 0.0596, 0.1977, 0.3651, 0.5549, 0.5272, 0.4268, 0.3478, 0.1820, 0.1600, 0.0366, 0.1036, 0.4838, 0.8075, 0.6585, 0.4435, 0.3562, 0.2014, 0.1192, 0.0534, 0.1260, 0.4336, 0.6904, 0.6846, 0.6177, 0.4702, 0.3483, 0.3138, 0.2453, 0.2144, 0.1114, 0.0837, 0.0335, 0.0214, 0.0356, 0.0758, 0.1778, 0.2354, 0.2254, 0.2484, 0.2207, 0.1470, 0.0528, 0.0424, 0.0131, 0.0000, 0.0073, 0.0262, 0.0638, 0.0727, 0.1851, 0.2395, 0.2150, 0.1574, 0.1250, 0.0816, 0.0345, 0.0209, 0.0094, 0.0445, 0.0868, 0.1898, 0.2594, 0.3358, 0.3504, 0.3708, 0.2500, 0.1438, 0.0445, 0.0690, 0.2976, 0.6354, 0.7233, 0.5397, 0.4482, 0.3379, 0.1919, 0.1266, 0.0560, 0.0785, 0.2097, 0.3216, 0.5152, 0.6522, 0.5036, 0.3483, 0.3373, 0.2829, 0.2040, 0.1077, 0.0350, 0.0225, 0.1187, 0.2866, 0.4906, 0.5010, 0.4038, 0.3091, 0.2301, 0.2458, 0.1595, 0.0853, 0.0382, 0.1966, 0.3870, 0.7270, 0.5816, 0.5314, 0.3462, 0.2338, 0.0889, 0.0591, 0.0649, 0.0178, 0.0314, 0.1689, 0.2840, 0.3122, 0.3332, 0.3321, 0.2730, 0.1328, 0.0685, 0.0356, 0.0330, 0.0371, 0.1862, 0.3818, 0.4451, 0.4079, 0.3347, 0.2186, 0.1370, 0.1396, 0.0633, 0.0497, 0.0141, 0.0262, 0.1276, 0.2197, 0.3321, 0.2814, 0.3243, 0.2537, 0.2296, 0.0973, 0.0298, 0.0188, 0.0073, 0.0502, 0.2479, 0.2986, 0.5434, 0.4215, 0.3326, 0.1966, 0.1365, 0.0743, 0.0303, 0.0873, 0.2317, 0.3342, 0.3609, 0.4069, 0.3394, 0.1867, 0.1109, 0.0581, 0.0298, 0.0455, 0.1888, 0.4168, 0.5983, 0.5732, 0.4644, 0.3546, 0.2484, 0.1600, 0.0853, 0.0502, 0.1736, 0.4843, 0.7929, 0.7128, 0.7045, 0.4388, 0.3630, 0.1647, 0.0727, 0.0230, 0.1987, 0.7411, 0.9947, 0.9665, 0.8316, 0.5873, 0.2819, 0.1961, 0.1459, 0.0534, 0.0790, 0.2458, 0.4906, 0.5539, 0.5518, 0.5465, 0.3483, 0.3603, 0.1987, 0.1804, 0.0811, 0.0659, 0.1428, 0.4838, 0.8127 }; REAL Mean; REAL TrainError; REAL TrainErrorPredictingMean; REAL TestError; REAL TestErrorPredictingMean; FILE* f; void NormalizeSunspots() { INT Year; REAL Min, Max; Min = MAX_REAL; Max = MIN_REAL; for (Year=0; YearAlpha = 0.5; Net->Eta = 0.05; Net->Gain = 1; NormalizeSunspots(); TrainErrorPredictingMean = 0; for (Year=TRAIN_LWB; Year<=TRAIN_UPB; Year++) { for (i=0; iLayer = (LAYER**) calloc(NUM_LAYERS, sizeof(LAYER*)); for (l=0; lLayer[l] = (LAYER*) malloc(sizeof(LAYER)); Net->Layer[l]->Units = Units[l]; Net->Layer[l]->Output = (REAL*) calloc(Units[l]+1, sizeof(REAL)); Net->Layer[l]->Error = (REAL*) calloc(Units[l]+1, sizeof(REAL)); Net->Layer[l]->Weight = (REAL**) calloc(Units[l]+1, sizeof(REAL*)); Net->Layer[l]->WeightSave = (REAL**) calloc(Units[l]+1, sizeof(REAL*)); Net->Layer[l]->dWeight = (REAL**) calloc(Units[l]+1, sizeof(REAL*)); Net->Layer[l]->Output[0] = BIAS; if (l != 0) { for (i=1; i<=Units[l]; i++) { Net->Layer[l]->Weight[i] = (REAL*) calloc(Units[l-1]+1, sizeof(REAL)); Net->Layer[l]->WeightSave[i] = (REAL*) calloc(Units[l-1]+1, sizeof(REAL)); Net->Layer[l]->dWeight[i] = (REAL*) calloc(Units[l-1]+1, sizeof(REAL)); } } } Net->InputLayer = Net->Layer[0]; Net->OutputLayer = Net->Layer[NUM_LAYERS - 1]; Net->Alpha = 0.9; Net->Eta = 0.25; Net->Gain = 1; } void RandomWeights(NET* Net) { INT l,i,j; for (l=1; lLayer[l]->Units; i++) { for (j=0; j<=Net->Layer[l-1]->Units; j++) { Net->Layer[l]->Weight[i][j] = RandomEqualREAL(-0.5, 0.5); } } } } void SetInput(NET* Net, REAL* Input) { INT i; for (i=1; i<=Net->InputLayer->Units; i++) { Net->InputLayer->Output[i] = Input[i-1]; } } void GetOutput(NET* Net, REAL* Output) { INT i; for (i=1; i<=Net->OutputLayer->Units; i++) { Output[i-1] = Net->OutputLayer->Output[i]; } } /****************************************************************************** S U P P O R T F O R S T O P P E D T R A I N I N G ******************************************************************************/ void SaveWeights(NET* Net) { INT l,i,j; for (l=1; lLayer[l]->Units; i++) { for (j=0; j<=Net->Layer[l-1]->Units; j++) { Net->Layer[l]->WeightSave[i][j] = Net->Layer[l]->Weight[i][j]; } } } } void RestoreWeights(NET* Net) { INT l,i,j; for (l=1; lLayer[l]->Units; i++) { for (j=0; j<=Net->Layer[l-1]->Units; j++) { Net->Layer[l]->Weight[i][j] = Net->Layer[l]->WeightSave[i][j]; } } } } /****************************************************************************** P R O P A G A T I N G S I G N A L S ******************************************************************************/ void PropagateLayer(NET* Net, LAYER* Lower, LAYER* Upper) { INT i,j; REAL Sum; for (i=1; i<=Upper->Units; i++) { Sum = 0; for (j=0; j<=Lower->Units; j++) { Sum += Upper->Weight[i][j] * Lower->Output[j]; } Upper->Output[i] = 1 / (1 + exp(-Net->Gain * Sum)); } } void PropagateNet(NET* Net) { INT l; for (l=0; lLayer[l], Net->Layer[l+1]); } } /****************************************************************************** B A C K P R O P A G A T I N G E R R O R S ******************************************************************************/ void ComputeOutputError(NET* Net, REAL* Target) { INT i; REAL Out, Err; Net->Error = 0; for (i=1; i<=Net->OutputLayer->Units; i++) { Out = Net->OutputLayer->Output[i]; Err = Target[i-1]-Out; Net->OutputLayer->Error[i] = Net->Gain * Out * (1-Out) * Err; Net->Error += 0.5 * sqr(Err); } } void BackpropagateLayer(NET* Net, LAYER* Upper, LAYER* Lower) { INT i,j; REAL Out, Err; for (i=1; i<=Lower->Units; i++) { Out = Lower->Output[i]; Err = 0; for (j=1; j<=Upper->Units; j++) { Err += Upper->Weight[j][i] * Upper->Error[j]; } Lower->Error[i] = Net->Gain * Out * (1-Out) * Err; } } void BackpropagateNet(NET* Net) { INT l; for (l=NUM_LAYERS-1; l>1; l--) { BackpropagateLayer(Net, Net->Layer[l], Net->Layer[l-1]); } } void AdjustWeights(NET* Net) { INT l,i,j; REAL Out, Err, dWeight; for (l=1; lLayer[l]->Units; i++) { for (j=0; j<=Net->Layer[l-1]->Units; j++) { Out = Net->Layer[l-1]->Output[j]; Err = Net->Layer[l]->Error[i]; dWeight = Net->Layer[l]->dWeight[i][j]; Net->Layer[l]->Weight[i][j] += Net->Eta * Err * Out + Net->Alpha * dWeight; Net->Layer[l]->dWeight[i][j] = Net->Eta * Err * Out; } } } } /****************************************************************************** S I M U L A T I N G T H E N E T ******************************************************************************/ void SimulateNet(NET* Net, REAL* Input, REAL* Output, REAL* Target, BOOL Training) { SetInput(Net, Input); PropagateNet(Net); GetOutput(Net, Output); ComputeOutputError(Net, Target); if (Training) { BackpropagateNet(Net); AdjustWeights(Net); } } void TrainNet(NET* Net, INT Epochs) { INT Year, n; REAL Output[M]; for (n=0; nError; } TestError = 0; for (Year=TEST_LWB; Year<=TEST_UPB; Year++) { SimulateNet(Net, &(Sunspots[Year-N]), Output, &(Sunspots[Year]), FALSE); TestError += Net->Error; } fprintf(f, "\nNMSE is %0.3f on Training Set and %0.3f on Test Set", TrainError / TrainErrorPredictingMean, TestError / TestErrorPredictingMean); } void EvaluateNet(NET* Net) { INT Year; REAL Output [M]; REAL Output_[M]; fprintf(f, "\n\n\n"); fprintf(f, "Year Sunspots Open-Loop Prediction Closed-Loop Prediction\n"); fprintf(f, "\n"); for (Year=EVAL_LWB; Year<=EVAL_UPB; Year++) { SimulateNet(Net, &(Sunspots [Year-N]), Output, &(Sunspots [Year]), FALSE); SimulateNet(Net, &(Sunspots_[Year-N]), Output_, &(Sunspots_[Year]), FALSE); Sunspots_[Year] = Output_[0]; fprintf(f, "%d %0.3f %0.3f %0.3f\n", FIRST_YEAR + Year, Sunspots[Year], Output [0], Output_[0]); } } /****************************************************************************** M A I N ******************************************************************************/ void main() { NET Net; BOOL Stop; REAL MinTestError; InitializeRandoms(); GenerateNetwork(&Net); RandomWeights(&Net); InitializeApplication(&Net); Stop = FALSE; MinTestError = MAX_REAL; do { TrainNet(&Net, 10); TestNet(&Net); if (TestError < MinTestError) { fprintf(f, " - saving Weights ..."); MinTestError = TestError; SaveWeights(&Net); } else if (TestError > 1.2 * MinTestError) { fprintf(f, " - stopping Training and restoring Weights ..."); Stop = TRUE; RestoreWeights(&Net); } } while (NOT Stop); TestNet(&Net); EvaluateNet(&Net); FinalizeApplication(&Net); } Simulator Output for the Time-Series Forecasting Application NMSE is 0.879 on Training Set and 0.834 on Test Set - saving Weights ... NMSE is 0.818 on Training Set and 0.783 on Test Set - saving Weights ... NMSE is 0.749 on Training Set and 0.693 on Test Set - saving Weights ... NMSE is 0.691 on Training Set and 0.614 on Test Set - saving Weights ... NMSE is 0.622 on Training Set and 0.555 on Test Set - saving Weights ... NMSE is 0.569 on Training Set and 0.491 on Test Set - saving Weights ... NMSE is 0.533 on Training Set and 0.467 on Test Set - saving Weights ... NMSE is 0.490 on Training Set and 0.416 on Test Set - saving Weights ... NMSE is 0.470 on Training Set and 0.401 on Test Set - saving Weights ... NMSE is 0.441 on Training Set and 0.361 on Test Set - saving Weights ... . . . NMSE is 0.142 on Training Set and 0.143 on Test Set NMSE is 0.142 on Training Set and 0.146 on Test Set NMSE is 0.141 on Training Set and 0.143 on Test Set NMSE is 0.146 on Training Set and 0.141 on Test Set NMSE is 0.144 on Training Set and 0.141 on Test Set NMSE is 0.140 on Training Set and 0.142 on Test Set NMSE is 0.144 on Training Set and 0.148 on Test Set NMSE is 0.140 on Training Set and 0.139 on Test Set - saving Weights ... NMSE is 0.140 on Training Set and 0.140 on Test Set NMSE is 0.141 on Training Set and 0.138 on Test Set - saving Weights ... . . . NMSE is 0.104 on Training Set and 0.154 on Test Set NMSE is 0.102 on Training Set and 0.160 on Test Set NMSE is 0.102 on Training Set and 0.160 on Test Set NMSE is 0.100 on Training Set and 0.157 on Test Set NMSE is 0.105 on Training Set and 0.153 on Test Set NMSE is 0.100 on Training Set and 0.155 on Test Set NMSE is 0.101 on Training Set and 0.154 on Test Set NMSE is 0.100 on Training Set and 0.158 on Test Set NMSE is 0.107 on Training Set and 0.170 on Test Set - stopping Training and restoring Weights ... NMSE is 0.141 on Training Set and 0.138 on Test Set Year Sunspots Open-Loop Prediction Closed-Loop Prediction 1960 0.572 0.532 0.532 1961 0.327 0.334 0.301 1962 0.258 0.158 0.146 1963 0.217 0.156 0.098 1964 0.143 0.236 0.149 1965 0.164 0.230 0.273 1966 0.298 0.263 0.405 1967 0.495 0.454 0.552 1968 0.545 0.615 0.627 1969 0.544 0.550 0.589 1970 0.540 0.474 0.464 1971 0.380 0.455 0.305 1972 0.390 0.270 0.191 1973 0.260 0.275 0.139 1974 0.245 0.211 0.158 1975 0.165 0.181 0.170 1976 0.153 0.128 0.175 1977 0.215 0.151 0.193 1978 0.489 0.316 0.274 1979 0.754 0.622 0.373