Mercurial > ift6266

annotate deep/convolutional_dae/scdae.py @ 282:698313f8f6e6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rajout de methode reliant toutes les couches cachees a la logistic et changeant seulement les parametres de la logistic durant finetune
author SylvainPL <sylvain.pannetier.lebeuf@umontreal.ca>
date Wed, 24 Mar 2010 14:45:02 -0400
parents 20ebc1f2a9fe
children 80ee63c3e749
rev   line source
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
1 from pynnet import *
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
2 # use hacks also
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
3 from pynnet.utils import *
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
4
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
5 import numpy
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
6 import theano
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
7 import theano.tensor as T
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
8
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
9 from itertools import izip
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
10
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
11 class cdae(LayerStack):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
12 def __init__(self, filter_size, num_filt, num_in, subsampling, corruption,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
13 dtype, img_shape):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
14 LayerStack.__init__(self, [ConvAutoencoder(filter_size=filter_size,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
15 num_filt=num_filt,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
16 num_in=num_in,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
17 noisyness=corruption,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
18 dtype=dtype,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
19 image_shape=img_shape),
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
20 MaxPoolLayer(subsampling)])
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
21
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
22 def build(self, input):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
23 LayerStack.build(self, input)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
24 self.cost = self.layers[0].cost
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
25
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
26 def cdae_out_size(in_size, filt_size, num_filt, num_in, subs):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
27 out = [None] * 3
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
28 out[0] = num_filt
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
29 out[1] = (in_size[1]-filt_size[0]+1)/subs[0]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
30 out[2] = (in_size[2]-filt_size[1]+1)/subs[1]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
31 return out
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
32
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
33 def scdae(in_size, num_in, filter_sizes, num_filts,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
34 subsamplings, corruptions, dtype):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
35 layers = []
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
36 old_nfilt = 1
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
37 for fsize, nfilt, subs, corr in izip(filter_sizes, num_filts,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
38 subsamplings, corruptions):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
39 layers.append(cdae(fsize, nfilt, old_nfilt, subs, corr, dtype,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
40 (num_in, in_size[0], in_size[1], in_size[2])))
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
41 in_size = cdae_out_size(in_size, fsize, nfilt, old_nfilt, subs)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
42 old_nfilt = nfilt
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
43 return LayerStack(layers), in_size
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
44
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
45 def mlp(layer_sizes, dtype):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
46 layers = []
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
47 old_size = layer_sizes[0]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
48 for size in layer_sizes[1:]:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
49 layers.append(SimpleLayer(old_size, size, activation=nlins.tanh,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
50 dtype=dtype))
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
51 old_size = size
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
52 return LayerStack(layers)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
53
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
54 def scdae_net(in_size, num_in, filter_sizes, num_filts, subsamplings,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
55 corruptions, layer_sizes, out_size, dtype, batch_size):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
56 rl1 = ReshapeLayer((None,)+in_size)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
57 ls, outs = scdae(in_size, num_in, filter_sizes, num_filts, subsamplings,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
58 corruptions, dtype)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
59 outs = numpy.prod(outs)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
60 rl2 = ReshapeLayer((None, outs))
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
61 layer_sizes = [outs]+layer_sizes
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
62 ls2 = mlp(layer_sizes, dtype)
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
63 lrl = SimpleLayer(layer_sizes[-1], out_size, activation=nlins.softmax)
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
64 return NNet([rl1, ls, rl2, ls2, lrl], error=errors.nll)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
65
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
66 def build_funcs(batch_size, img_size, filter_sizes, num_filters, subs,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
67 noise, mlp_sizes, out_size, dtype, pretrain_lr, train_lr):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
68
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
69 n = scdae_net((1,)+img_size, batch_size, filter_sizes, num_filters, subs,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
70 noise, mlp_sizes, out_size, dtype, batch_size)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
71 x = T.fmatrix('x')
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
72 y = T.ivector('y')
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
73
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
74 def pretrainfunc(net, alpha):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
75 up = trainers.get_updates(net.params, net.cost, alpha)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
76 return theano.function([x], net.cost, updates=up)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
77
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
78 def trainfunc(net, alpha):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
79 up = trainers.get_updates(net.params, net.cost, alpha)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
80 return theano.function([x, y], net.cost, updates=up)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
81
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
82 n.build(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
83 pretrain_funcs_opt = [pretrainfunc(l, pretrain_lr) for l in n.layers[1].layers]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
84 trainf_opt = trainfunc(n, train_lr)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
85 evalf_opt = theano.function([x, y], errors.class_error(n.output, y))
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
86
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
87 clear_imgshape(n)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
88 n.build(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
89 pretrain_funcs_reg = [pretrainfunc(l, 0.01) for l in n.layers[1].layers]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
90 trainf_reg = trainfunc(n, 0.1)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
91 evalf_reg = theano.function([x, y], errors.class_error(n.output, y))
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
92
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
93 def select_f(f1, f2, bsize):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
94 def f(x):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
95 if x.shape[0] == bsize:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
96 return f1(x)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
97 else:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
98 return f2(x)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
99 return f
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
100
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
101 pretrain_funcs = [select_f(p_opt, p_reg, batch_size) for p_opt, p_reg in zip(pretrain_funcs_opt, pretrain_funcs_reg)]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
102
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
103 def select_f2(f1, f2, bsize):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
104 def f(x, y):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
105 if x.shape[0] == bsize:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
106 return f1(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
107 else:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
108 return f2(x, y)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
109 return f
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
110
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
111 trainf = select_f2(trainf_opt, trainf_reg, batch_size)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
112 evalf = select_f2(evalf_opt, evalf_reg, batch_size)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
113 return pretrain_funcs, trainf, evalf
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
114
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
115 def do_pretrain(pretrain_funcs, pretrain_epochs):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
116 for f in pretrain_funcs:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
117 for i in xrange(pretrain_epochs):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
118 f()
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
119
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
120 def massage_funcs(train_it, dset, batch_size, pretrain_funcs, trainf, evalf):
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
121 def pretrain_f(f):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
122 def res():
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
123 for x, y in train_it:
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
124 yield f(x)
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
125 it = res()
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
126 return lambda: it.next()
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
127
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
128 pretrain_fs = map(pretrain_f, pretrain_funcs)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
129
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
130 def train_f(f):
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
131 def dset_it():
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
132 for x, y in train_it:
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
133 yield f(x, y)
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
134 it = dset_it()
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
135 return lambda: it.next()
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
136
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
137 train = train_f(trainf)
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
138
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
139 def eval_f(f, dsetf):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
140 def res():
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
141 c = 0
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
142 i = 0
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
143 for x, y in dsetf(batch_size):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
144 i += x.shape[0]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
145 c += f(x, y)*x.shape[0]
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
146 return c/i
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
147 return res
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
148
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
149 test = eval_f(evalf, dset.test)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
150 valid = eval_f(evalf, dset.valid)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
151
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
152 return pretrain_fs, train, valid, test
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
153
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
154 def repeat_itf(itf, *args, **kwargs):
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
155 while True:
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
156 for e in itf(*args, **kwargs):
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
157 yield e
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
158
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
159 def run_exp(state, channel):
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
160 from ift6266 import datasets
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
161 from sgd_opt import sgd_opt
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
162 import sys, time
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
163
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
164 channel.save()
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
165
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
166 # params: bsize, pretrain_lr, train_lr, nfilts1, nfilts2, nftils3, nfilts4
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
167 # pretrain_rounds
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
168
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
169 dset = dataset.nist_all()
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
170
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
171 nfilts = []
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
172 if state.nfilts1 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
173 nfilts.append(state.nfilts1)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
174 if state.nfilts2 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
175 nfilts.append(state.nfilts2)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
176 if state.nfilts3 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
177 nfilts.append(state.nfilts3)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
178 if state.nfilts4 != 0:
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
179 nfilts.append(state.nfilts4)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
180
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
181 fsizes = [(5,5)]*len(nfilts)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
182 subs = [(2,2)]*len(nfilts)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
183 noise = [state.noise]*len(nfilts)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
184
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
185 pretrain_funcs, trainf, evalf = build_funcs(
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
186 img_size=(32, 32),
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
187 batch_size=state.bsize,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
188 filter_sizes=fsizes,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
189 num_filters=nfilts,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
190 subs=subs,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
191 noise=noise,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
192 mlp_sizes=[state.mlp_sz],
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
193 out_size=62,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
194 dtype=numpy.float32,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
195 pretrain_lr=state.pretrain_lr,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
196 train_lr=state.train_lr)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
197
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
198 pretrain_fs, train, valid, test = massage_funcs(
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
199 state.bsize, dset, pretrain_funcs, trainf, evalf)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
200
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
201 do_pretrain(pretrain_fs, state.pretrain_rounds)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
202
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
203 sgd_opt(train, valid, test, training_epochs=100000, patience=10000,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
204 patience_increase=2., improvement_threshold=0.995,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
205 validation_frequency=2500)
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
206
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
207 if __name__ == '__main__':
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
208 from ift6266 import datasets
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
209 from sgd_opt import sgd_opt
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
210 import sys, time
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
211
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
212 batch_size = 100
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
213 dset = datasets.mnist()
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
214
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
215 pretrain_funcs, trainf, evalf = build_funcs(
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
216 img_size = (28, 28),
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
217 batch_size=batch_size, filter_sizes=[(5,5), (3,3)],
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
218 num_filters=[4, 4], subs=[(2,2), (2,2)], noise=[0.2, 0.2],
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
219 mlp_sizes=[500], out_size=10, dtype=numpy.float32,
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
220 pretrain_lr=0.01, train_lr=0.1)
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
221
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
222 pretrain_fs, train, valid, test = massage_funcs(
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
223 repeat_itf(dset.train, batch_size),
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
224 dset, batch_size,
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
225 pretrain_funcs, trainf, evalf)
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
226
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
227 print "pretraining ...",
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
228 sys.stdout.flush()
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
229 start = time.time()
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
230 do_pretrain(pretrain_fs, 2500)
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
231 end = time.time()
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
232 print "done (in", end-start, "s)"
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
233
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
234 sgd_opt(train, valid, test, training_epochs=10000, patience=1000,
276
727ed56fad12 Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff changeset
235 patience_increase=2., improvement_threshold=0.995,
277
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
236 validation_frequency=250)
20ebc1f2a9fe Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents: 276
diff changeset
237