View file src/neur/tmgrptrrev.ijs - Download

echo 'Test with MNIST data'
load 'bp.ijs'
echo 'Load data'
load 'labels.ijs'
load 'images.ijs'
echo 'Data loaded'

ntrain =: 2000
ntest =: 100

nrtrain =: 200
nrtest =: 10

NB. Train with 80 first items
outputs =: |: (ntrain {. labels) =/ i. 10
inputs =: |: (ntrain {. images) % 256

outputs =: (outputs,0) ,. ((10#0),1) */ nrtrain#1
inputs =: inputs ,. (? ((#inputs),nrtrain) $ 256) % 256

NB. Train
NB. echo 'Learning ...'
NB. brain =: learn inputs ; outputs ; (3 # 15) ; 7 ; 3000 ; 10 ; 0.00001
learning =: DefaultLearning
learning =: learning LearningInputs  inputs
learning =: learning LearningOutputs outputs
learning =: learning LearningLayers  3 # 15
learning =: learning LearningRate    8
learning =: learning LearningSteps   3000 
learning =: learning LearningError   (ntrain+nrtrain) * 0.1
NB. brain =: learn learning

NB. echo 'Saving ...'
NB. (3!:1 brain) 1!:2 < 'brainrnd.jdata'

echo 'Load pretrained brain'
brain =: 3!:2 (1!:1 < 'brainrnd.jdata')
echo 'Brain loaded'

W =: > 0 { brain
B =: > 1 { brain
nl =: > 2 { brain

NB. Test with training data
result =: brain apply inputs
got =: result OutputsGot learning
echo 'Errors with training data : ', ": +/ +/ 0.1 < | got - outputs

NB. Test data : items 100 - 119

outtest =: |: (ntest {. ntrain }. labels) =/ i. 10
intest =: |: (ntest {. ntrain }. images) % 256

outtest =: (outtest,0) ,. ((10#0),1) */ nrtest#1
intest =: intest ,. (? ((#intest),nrtest) $ 256) % 256

result =: brain apply intest
got =: result OutputsGot learning
echo 'Errors with test data : ', ": +/ +/ 0.1 < | got - outtest
echo 'Expected : ', ": +/ outtest * i. 11
echo 'Got      : ', ": +/ (got = (11 # 1) */ >./ got) * i. 11
echo 'Success  : ', ": +/ (+/ outtest * i. 11 ) = +/ (got = (11 # 1) */ >./ got) * i. 11


NB. Reverse network

ninputs =: # inputs
noutputs =: # outputs
ntotal =: # B

NB. Permutation : outputs at the beginning and inputs at the end
permute =: ((ntotal-noutputs) + i. noutputs) , (ninputs + i. ntotal-ninputs+noutputs) , (i. ninputs)

revW =: permute { permute {"1 |: W
revB =: permute { B
revnl =: nl

revbrain =: revW ; revB ; revnl

revinputs =: (i. 10) =/ i. 10
revresult =: revbrain apply revinputs
 
A =: > 1 { revresult
n =: # A
no =: ninputs
revgot =: ((no $ n-no) + i. no) { A
avg =: (+/ |: revgot) % 10


ni =: # inputs
no =: # outputs
nn =: # B

NB. Random input image
ingen =: (ni,1) $ (? ni $ 256) % 256
ingeninit =: ingen
resgeninit =: brain apply ingen
gotgeninit =: resgeninit OutputsGot learning

nstep =: 0
digit =: 7
initrate =: 8
ratevar =: 2
b =: _1
a =: 2
nloop =: 30

NB. Computation of the partial derivatives of outputs with respect to inputs
NB. Example with one input larer (i), 3 intermediate layers (j,k,l) and ono output layer (m)
NB. Notation : Sl = sum over l
NB. dam/dai = dam/dzm dzm/dai = s'(zm) dzm/dai
NB. dzm/dai = Sl dzm/dal dal/dai = Sl wml dal/dai
NB. dam/dai = s'(zm) Sl wml dal/dai
NB. dal/dai = dal/dzl dzl/dai = s'(zl) dzl/dai
NB. dzl/dai = Sk dzl/dak dak/dai = Sk wlk dak/dai
NB. dal/dai = s'(zl) Sk wlk dak/dai
NB. dam/dai = s'(zm) Sl wml s'(zl) Sk wlk dak/dai
NB. dak/dai = dak/dzk dzk/dai = s'(zk) dzk/dai
NB. dzk/dai = Sj dzk/daj daj/dai = Sj wkj daj/dai
NB. dak/dai = s'(zk) Sj wkj daj/dai
NB. dam/dai = s'(zm) Sl wml s'(zl) Sk wlk s'(zk) Sj wkj daj/dai
NB. daj/dai = daj/dzj dzj/dai = s'(zj) dzj/dai
NB. dzj/dai = wji
NB. daj/dai = s'(zj) wji
NB. dam/dai = s'(zm) Sl wml s'(zl) Sk wlk s'(zk) Sj wkj s'(zj) wji
NB. Matrix of dam/dai = (sigmaprime ,Z) * W +/ . * (sigmaprime ,Z) * W +/ . * (sigmaprime ,Z) * W +/ . * (sigmaprime ,Z) * W 
NB.                   = ({{ (sigmaprime ,Z) * W +/ . * y }} ^: 3) (sigmaprime ,Z) * W 

NB. Check for pixel p if calculated and experiemental gradients are almost the same

p =: 12 

NB. Calculated gradient
resgen =: brain apply ingen
gotgen =: resgen OutputsGot learning
Z =: > 0 { resgen
D =: ({{ (sigmaprime ,Z) * W +/ . * y }} ^: (nl-2)) (sigmaprime ,Z) * W
D1 =: ni ({."1) ((nn-no) + i. no) { D
calc =: 11 1 $ p {"1 D1  NB. Calculated gradient

epsilon =: 0.001

NB. Experimental gradient
NB. ingen gp p = gradient for pixel p
gp =: 4 : 0
 in =. x
 p =. y
 res =. brain apply in
 got =. res OutputsGot learning
 in1 =. (ni,1) $ (,in) + epsilon * p = i. ni
 res1 =. brain apply in1
 got1 =. res1 OutputsGot learning
 (1 % epsilon) * got1 - got
)

exp =: ingen gp p    NB. Experimental gradient

echo 'Difference between calculated and experimantal gradient'
echo calc - exp      NB. Difference

NB. stepgen =: 3 : 0
loopgen =: 3 : 0

rate =: initrate
NB. ingen =: ingeninit
ingen =: (ni,1) $ digit { |: revgot
nstep =: 0

for. i. y do.
 nstep =: nstep + 1
 NB. echo nstep

 resgen =: brain apply ingen
 gotgen =: resgen OutputsGot learning
 value =: +/ (,gotgen) * b + a * digit = i. 11
 NB. echo 'Step ', (": nstep), ': value = ', (": value)
 
 ringen =: (ni,1) $ (? ni $ 256) % 256
 rresgen =: brain apply ringen
 rgotgen =: rresgen OutputsGot learning
 rvalue =: +/ (,rgotgen) * b + a * digit = i. 11
 if. rvalue > value do.
  ingen =: ringen
  resgen =: rresgen
  gotgen =: rgotgen
  value =: rvalue
 end.

 Z =: > 0 { resgen
 A =: > 1 { resgen

 NB. Matrix of partial derivatives of outputs with respect to inputs
 D =: ({{ (sigmaprime ,Z) * W +/ . * y }} ^: (nl-2)) (sigmaprime ,Z) * W 
 D1 =: ni ({."1) ((nn-no) + i. no) { D
 
 coefs =: b + a * digit = i. 11
 var =: 784 1 $ +/ coefs * D1
 ningen =: ingen + rate * var
 min =: <./ <./ ningen
 max =: >./ >./ ningen 
 ningen =: 0 >. 1 <. ningen

 nresgen =: brain apply ningen
 ngotgen =: nresgen OutputsGot learning
 nvalue =: +/ (,ngotgen) * b + a * digit = i. 11
 echo 'Step ', (": nstep), ' : rate = ', (": rate),  ' ; value = ', (": value), ' ; rvalue = ', (": rvalue), ' ; nvalue = ',  (": nvalue), ' ; min = ', (": min), ' ; max = ',(": max)
 NB. echo 'Step ', (": nstep), ' : rate = ', (": rate),  ' ; value = ', (": value), ' ; nvalue = ',  (": nvalue), ' ; min = ', (": min), ' ; max = ',(": max)

 if. nvalue > value do.
  ingen =: ningen
  rate =: rate * ratevar
 else.
  rate =: rate % ratevar
 end.
 
end.
)

echo 'rate = ', (": initrate), ' ; ratevar = ', (": ratevar), ' ; nloop = ', (": nloop)
echo ' '

gen =: 3 : 0
 digit =: y
 echo 'digit = ', ": digit

 NB. (stepgen ^: 20) 0
 loopgen nloop

 resgenterm =: brain apply ingen
 gotgenterm =: resgenterm OutputsGot learning

 NB. echo 'Difference between final and initial outputs'
 NB. echo gotgenterm - gotgeninit
 NB. echo ' '
 echo 'Final outputs'
 echo (i. 11) ,. gotgenterm
 termvalue =: +/ (,gotgenterm) * b + a * digit = i. 11
 echo 'Final value : ', (": termvalue)
 echo ' '
 echo (0.5 < 28 28 $ , ingen) { 1 88
 echo ' '
 NB. (3!:1 ingen) 1!:2 < 'digit.jdata'
)

genall =: 3 : 0
 for_i. i. 10 do.
  gen i
 end.
 )
 
genall 0