View file src/neur/gradient.ijs - Download
load 'bp.ijs'
NB. Initialization :
NB. makebrain layers; 0
NB. Make brain from flat parameters array P :
NB. makebrain layers; 1; P
NB. where layers is the array of numbers of neurons per layer
NB. gives W ; B ; nl ; P
NB. where :
NB. W is the matrix of connection weights
NB. B is the vector of biases
NB. nl is the number of layers
NB. P is the vector of parameters (non zero elements of W and B)
makebrain =: 3 : 0
layers =. > 0 { y NB. Number of neurons of layers
init =. > 1 { y NB. 1 if initialization asked, 0 to use given parameters array
nl =. # layers NB. Number of layers
nn =. +/ layers NB. Number of neurons
ln =. layers # i. nl
maskW =. ln =/ 1 + ln NB. Mask of weights matrix
maskB =. ln > 0 NB. Mask of biases vector
npW =. +/ +/ maskW NB. Number of parameters of weights matrix
npB =. +/ maskB NB. Number of parameters of biases matrix
np =. npW + npB NB. Total number of parameters
if. init do.
P =. normalrand np NB. Initialize parameters array randomly
else.
P =. > 2 { y NB. Get given parameters array
end.
PW =. (i. npW) { P NB. Parameters for weight matrix
PB =. (npW + i. npB) { P NB. Parameters for biases matrix
W =. (nn,nn) $ ((,maskW) * +/\ ,maskW) { 0, PW NB. Distribute parameters of PW in W according to maskW
B =. (maskB * +/\ maskB) { 0, PB NB. Distribute parameters of PB in B according to maskB
W ; B ; nl ; P
)
NB. Compute the loss for a given parameters array P
NB. (layers; inputs; outputs) computeloss P
NB. Result : the computed loss
computeloss =: 4 : 0 "_ 1
layers =. > 0 { x
inputs =. > 1 { x
outputs =. > 2 { x
P =. y
no =. # outputs
nn =. +/ layers
nX =. 1 { $ inputs NB. Number of inputs
brain =. makebrain layers; 0; P
result =. brain apply inputs
out =. ((no $ nn-no) + i. no) { > 1 { result
err =. out - outputs
loss =. (+/ +/ *: err) % 2 * nX
loss
)
p =: 10
inputs =: (? (8,3) $ p) % p
outputs =: (? (5,3) $ p) % p
ni =: # inputs
no =: # outputs
layers =: ni, 4 4 4, no
nn =: +/ layers
brain =: makebrain layers; 1
P =: > 3 { brain
result =: brain apply inputs
out =: ((no $ nn-no) + i. no) { > 1 { result
err1 =: out - outputs
echo err1
echo 'Loss = ', ": +/ +/ *: err1
brain =: makebrain layers; 0; P
result =: brain apply inputs
out =: ((no $ nn-no) + i. no) { > 1 { result
err2 =: out - outputs
echo err2-err1
loss =: (layers; inputs; outputs) computeloss P
echo 'Loss = ', ": loss
eps =: 0.0001
nP =: # P
Pplus =: |: P + eps * (i. nP) =/ i. nP
Pminus =: |: P - eps * (i. nP) =/ i. nP
NB. gradient =: (1%2*eps) * ((layers; inputs; outputs) computeloss Pplus) - ((layers; inputs; outputs) computeloss Pminus)
gradient =. (1%2*eps) * ((layers; inputs; outputs) computeloss Pplus) - ((layers; inputs; outputs) computeloss Pminus)
echo gradient
NB. Gradient descent
NB. descent learning P
NB. learning : learning parameters
NB. P : brain flat parameters array
NB. Result : optimized flat parameters array
descent =: 3 : 0
learning =. > 0 { y
P =. > 1 { y
inputs =. LearningInputs learning
outputs =. LearningOutputs learning
nnl =. LearningLayers learning NB. Number of neurons of intermediate layers
alpha =. LearningRate learning NB. Learning rate
nls =. LearningSteps learning NB. Learning steps
targetloss =. (LearningError learning) % 2 * 1 { $ inputs
de =. (LearningVariation learning) % 2 * 1 { $ inputs
layers =. (# inputs), nnl, (# outputs) NB. number of neurons of all layers
nP =. # P NB. Number of brain parameters
eps =. 0.0001
step =. 0
NB. while. loss > targetloss do.
for. i. nls do.
step =. step + 1
loss1 =. loss
loss =. (layers; inputs; outputs) computeloss P
echo 'Step ', (": step), ' : loss = ', ": loss
if. (loss < targetloss) +. de > | loss - loss1 do. break. end.
Pplus =. |: P + eps * (i. nP) =/ i. nP
Pminus =. |: P - eps * (i. nP) =/ i. nP
gradient =. (1%2*eps) * ((layers; inputs; outputs) computeloss Pplus) - ((layers; inputs; outputs) computeloss Pminus)
P =. P - alpha * gradient
end.
P
)
NB. learn learning
NB. learning : learning parameters
NB. Result : brain = W ; B ; nl ; P
learn =: 3 : 0
learning =: y
inputs =. LearningInputs y
outputs =. LearningOutputs y
nnl =. LearningLayers y NB. Numbers of neurons in the intermediate layers
NB. alpha =. LearningRate y NB. Learning rate
NB. nls =. LearningSteps y NB. Number of learning steps
NB. er =. LearningError y NB. Maximum error allowed
NB. de =. LearningVariation y NB. Stop learning when variation of error less than it
ni =. # inputs NB. Number of inputs
no =. # outputs NB. Number of outputs
nil =. # nnl NB. Number of intermediate layers
nl =. nil + 2 NB. Total number of layers
n =. ni + no + +/nnl NB. Number of neurons
nX =. 1 { $ inputs NB. Number of inputs
layers =. ni, nnl, no NB. Number of neurons of all layers
brain =: makebrain layers; 1 NB. Initialize brain
P =: > 3 { brain NB. Get flat parameters array
P =. descent learning; P NB. Gradient descent
brain =. makebrain layers; 0; P
brain
)
NB. Test with random inputs and expected outputs
testrandom =: 3 : 0
p =: 10
nl =: 5 NB. Number of layers
inputs =: (? (8,3) $ p) % p
outputs =: (? (5,3) $ p) % p
NB. brain =: learn inputs ; outputs ; ((nl-2) # 4) ; 16 ; 3000 ; 0.001 ; 0.000001
learning =: DefaultLearning
learning =: learning LearningInputs inputs
learning =: learning LearningOutputs outputs
learning =: learning LearningLayers ((nl-2) # 4)
learning =: learning LearningRate 16
learning =: learning LearningSteps 3000
learning =: learning LearningError 0.001
learning =: learning LearningVariation 0.0000001
brain =: learn learning
echo ' '
echo 'Test with random inputs and expected outputs :'
result =: brain apply inputs
got =: result OutputsGot learning
echo 'Expected :'
echo outputs
echo 'Got :'
echo got
echo 'Errors :'
echo got - outputs
)
testand =: 3 : 0
init 0
0 0 0 gives 0
0 1 0 gives 0
1 0 0 gives 0
1 1 0 gives 1
NB. brain =: learn inputs ; outputs ; (3 # 3) ; 16 ; 3000 ; 0.001 ; 0.000001
learning =: DefaultLearning
learning =: learning LearningInputs inputs
learning =: learning LearningOutputs outputs
learning =: learning LearningLayers (3 # 3)
learning =: learning LearningRate 16
learning =: learning LearningSteps 3000
learning =: learning LearningError 0.001
brain =: learn learning
echo ' '
echo 'Test with logical and :'
result =: brain apply inputs
got =: result OutputsGot learning
echo 'Expected :'
echo outputs
echo 'Got :'
echo got
echo 'Errors :'
echo got - outputs
init 0
0 0 1 gives 0
0 1 1 gives 0
1 0 1 gives 0
1 1 1 gives 0
echo ' '
echo 'Test with logical and : outputs ='
echo (> 1 { brain apply inputs)
echo ' '
)