View file src/neur/testmnistgen.ijs - Download
echo 'Test with MNIST data'
load 'bp.ijs'
echo 'Load data'
load 'labels.ijs'
load 'images.ijs'
echo 'Data loaded'
ntrain =: 500
ntest =: 100
NB. Train with 80 first items
outputs =: |: (ntrain {. labels) =/ i. 10
inputs =: |: (ntrain {. images) % 256
NB. Train
NB. echo 'Learning ...'
NB. brain =: learn inputs ; outputs ; (3 # 15) ; 7 ; 3000 ; 10 ; 0.00001
learning =: DefaultLearning
learning =: learning LearningInputs inputs
learning =: learning LearningOutputs outputs
learning =: learning LearningLayers 3 # 15
learning =: learning LearningRate 8
learning =: learning LearningSteps 3000
learning =: learning LearningError ntrain * 0.1
NB. brain =: learn learning
NB. echo 'Saving ...'
NB. (3!:1 brain) 1!:2 < 'brain.jdata'
echo 'Load pretrained brain'
brain =: 3!:2 (1!:1 < 'brain.jdata')
echo 'Brain loaded'
W =: > 0 { brain
B =: > 1 { brain
nl =: > 2 { brain
NB. Test with training data
result =: brain apply inputs
got =: result OutputsGot learning
echo 'Errors with training data : ', ": +/ +/ 0.1 < | got - outputs
NB. Test data : items 100 - 119
outtest =: |: (ntest {. ntrain }. labels) =/ i. 10
intest =: |: (ntest {. ntrain }. images) % 256
result =: brain apply intest
got =: result OutputsGot learning
echo 'Errors with test data : ', ": +/ +/ 0.1 < | got - outtest
echo 'Expected : ', ": +/ outtest * i. 10
echo 'Got : ', ": +/ (got = (10 # 1) */ >./ got) * i. 10
echo 'Success : ', ": +/ (+/ outtest * i. 10 ) = +/ (got = (10 # 1) */ >./ got) * i. 10
ni =: # inputs
no =: # outputs
nn =: # B
NB. Random input image
ingen =: (ni,1) $ (? ni $ 256) % 256
resgeninit =: brain apply ingen
gotgeninit =: resgeninit OutputsGot learning
nstep =: 0
digit =: 0
rate =: 0.1
b =: _1
a =: 2
NB. Computation of the partial derivatives of outputs with respect to inputs
NB. Example with one input larer (i), 3 intermediate layers (j,k,l) and ono output layer (m)
NB. Notation : Sl = sum over l
NB. dam/dai = dam/dzm dzm/dai = s'(zm) dzm/dai
NB. dzm/dai = Sl dzm/dal dal/dai = Sl wml dal/dai
NB. dam/dai = s'(zm) Sl wml dal/dai
NB. dal/dai = dal/dzl dzl/dai = s'(zl) dzl/dai
NB. dzl/dai = Sk dzl/dak dak/dai = Sk wlk dak/dai
NB. dal/dai = s'(zl) Sk wlk dak/dai
NB. dam/dai = s'(zm) Sl wml s'(zl) Sk wlk dak/dai
NB. dak/dai = dak/dzk dzk/dai = s'(zk) dzk/dai
NB. dzk/dai = Sj dzk/daj daj/dai = Sj wkj daj/dai
NB. dak/dai = s'(zk) Sj wkj daj/dai
NB. dam/dai = s'(zm) Sl wml s'(zl) Sk wlk s'(zk) Sj wkj daj/dai
NB. daj/dai = daj/dzj dzj/dai = s'(zj) dzj/dai
NB. dzj/dai = wji
NB. daj/dai = s'(zj) wji
NB. dam/dai = s'(zm) Sl wml s'(zl) Sk wlk s'(zk) Sj wkj s'(zj) wji
NB. Matrix of dam/dai = (sigmaprime ,Z) * W +/ . * (sigmaprime ,Z) * W +/ . * (sigmaprime ,Z) * W +/ . * (sigmaprime ,Z) * W
NB. = ({{ (sigmaprime ,Z) * W +/ . * y }} ^: 3) (sigmaprime ,Z) * W
NB. Check for pixel p if calculated and experiemental gradients are almost the same
p =: 12
NB. Calculated gradient
resgen =: brain apply ingen
gotgen =: resgen OutputsGot learning
Z =: > 0 { resgen
D =: ({{ (sigmaprime ,Z) * W +/ . * y }} ^: (nl-2)) (sigmaprime ,Z) * W
D1 =: ni ({."1) ((nn-no) + i. no) { D
calc =: 10 1 $ p {"1 D1 NB. Calculated gradient
epsilon =: 0.001
NB. Experimental gradient
NB. ingen gp p = gradient for pixel p
gp =: 4 : 0
in =. x
p =. y
res =. brain apply in
got =. res OutputsGot learning
in1 =. (ni,1) $ (,in) + epsilon * p = i. ni
res1 =. brain apply in1
got1 =. res1 OutputsGot learning
(1 % epsilon) * got1 - got
)
exp =: ingen gp p NB. Experimental gradient
echo 'Difference between calculated and experimantal gradient'
echo calc - exp NB. Difference
stepgen =: 3 : 0
nstep =: nstep + 1
NB. echo nstep
resgen =: brain apply ingen
gotgen =: resgen OutputsGot learning
value =: +/ (,gotgen) * b + a * digit = i. 10
NB. echo 'Step ', (": nstep), ': value = ', (": value)
ringen =: (ni,1) $ (? ni $ 256) % 256
rresgen =: brain apply ringen
rgotgen =: rresgen OutputsGot learning
rvalue =: +/ (,rgotgen) * b + a * digit = i. 10
echo 'Step ', (": nstep), ' : value = ', (": value), ' ; rvalue = ', (": rvalue)
if. rvalue > value do.
ingen =: ringen
resgen =: rresgen
gotgen =: rgotgen
end.
Z =: > 0 { resgen
A =: > 1 { resgen
NB. Matrix of partial derivatives of outputs with respect to inputs
D =: ({{ (sigmaprime ,Z) * W +/ . * y }} ^: (nl-2)) (sigmaprime ,Z) * W
D1 =: ni ({."1) ((nn-no) + i. no) { D
coefs =: b + a * digit = i. 10
var =: 784 1 $ +/ coefs * D1
ingen =: ingen + rate * var
)
echo 'rate = ', ": rate
echo 'digit = ', ": digit
(stepgen ^: 10) 0
resgenterm =: brain apply ingen
gotgenterm =: resgenterm OutputsGot learning
echo 'Difference between final and initial outputs'
echo gotgenterm - gotgeninit
echo ' '
echo 'Final outputs'
echo gotgenterm
echo ' '
echo (0.7 < 28 28 $ , ingen) { 1 88