% fapprox.m
% Date: 2/22/09
% Modified for new Version 6 Toolbox
% Author: G. Dempsey, Bradley Univ.- Electrical Engineering 
% FUNCTION APPROXIMATION Example
%
% Two Layer Network: 1 input-1 output, Hidden layer sigmoid, Output layer linear
% trained with backpropagation on a 21 pattern problem.
clc
close all
clear

% PROBLEM DEFINITION
%===================

% Define twenty-one 1-element input vectors.

P = -1:.1:1;

% Define the associated twenty-one 1-element targets.

T = [-.9602 -.5770 -.0729  .3771  .6405  .6600  .4609 ...
      .1336 -.2013 -.4344 -.5000 -.3930 -.1647  .0988 ...
      .3072  .3960  .3449  .1816 -.0312 -.2189 -.3201];

% Define eighty-one 1-element input vectors 
% for our test set.

P2 = -1:.025:1;

% PLOT TRAINING VECTORS
%======================
disp('Press key to show Training Vectors')
pause

plot(P,T,'+');
title('Training Vectors');
xlabel('Input Vector P');
ylabel('Target Vector T');

% INITIALIZE NETWORK ARCHITECTURE
%================================

% Set input vector size R, layer sizes S1 & S2.

S1=5;		% 5 hidden neurons
S2=1;		% single output neuron 
R=1;		% number of input components, note parameter
			% not used by training fcn, but good documentation

% Initialize weights and biases. Nyguyen/Widrow method is the default
% initialize the network with single command
% function newff for Version 5.0 Toolbox: enclosed in ***********
%****************************************************************
% net=newff([-1 1],[S1,S2],{'tansig','purelin'},'traingdx');
%****************************************************************

% initialize the network with single command
% function newff for Version 6 Toolbox: enclosed in ***********
%*********************************************************************
net=newff(P,T,S1,{'tansig','purelin'},'traingdx');    % BP with adaptive learning and momentum
net.divideFcn = '';                 % don't divide input data into test and validation data
net.inputs{1}.processFcns={};       % don't perform advanced preprocessing
net.outputs{2}.processFcns={};      % don't perform advanced postprocessing
%net.trainParam.showWindow = false; % this will disable TRAIN GUI
%net.trainParam.showCommandLine = true; % this will enable status in command window
%**********************************************************************

% save a copy of the weights/biases from newff
W10=net.IW{1,1};
B10=net.b{1};
W20=net.LW{2,1};
B20=net.b{2};

% PLOT INITIAL FUNCTION APPROXIMATION
%====================================
A=sim(net,P);

disp('plot initial approximation, press key');
pause
clf
plot(P,T,'+');
hold
plot(P,A,'r');
title('Initial Function Approximation');
xlabel('Input Vector P');
ylabel('ANN Output');
hold
disp('press key to start training')
pause

% TRAIN THE NETWORK
%==================

% TRAINING PARAMETERS
net.trainParam.show=50;			% display update
net.trainParam.lr=0.01;			% learning rate
net.trainParam.epochs=2000;		% max epochs
net.trainParam.goal=0.001;		% error goal
net.trainParam.lr_inc=1.05;		% lr increase
net.trainParam.lr_dec=0.7;		% lr decrease
net.trainParam.mc=0.95;			% momentum
net.trainParam.max_perf_inc=1.04;	% error ratio check for weight discard

% Now train the ANN
[net,tr]=train(net,P,T);

% summarize 

fprintf('\nFINAL NETWORK VALUES:\n')
W1=net.IW{1,1}	% input to hidden layer weights
B1=net.b{1}		% hidden neuron biases
W2=net.LW{2,1}	% hidden to output layer weights
B2=net.b{2}		% output neuron bias

A=sim(net,P2);	% calculate ANN output with test set

fprintf('Trained for %.0f epochs.\n',max(tr.epoch))
fprintf('Mean square error goal was %g.\n',net.trainParam.goal)
fprintf('Final mean square error is %g.\n',tr.perf(max(tr.epoch+1)));
fprintf('Trained network operates: ');
if tr.perf(max(tr.epoch+1))< net.trainParam.goal
  disp('It works!')
else
  disp('It Failed!')
  end

% PLOT FINAL APPROXIMATION
%=========================
disp('plot final approximation: press key');
pause
figure					% open new figure
plot(P,T,'+');			% training data points
hold
plot(P2,A,'r');		% test set output
title('Final Function Approximation');
xlabel('Input Vector P');
ylabel('ANN Output');
hold

% PLOT Adaptive learning rate CURVE
%=================
disp('plot learning rate curve: press key');
pause
figure
plot(tr.lr);
title('Adaptive Learning Rate Versus Epochs');
xlabel('Epochs');
ylabel('Learning Rate');