Mercurial > ift6266

comparison deep/convolutional_dae/scdae.py @ 276:727ed56fad12

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add reworked code for convolutional auto-encoder.
author Arnaud Bergeron <abergeron@gmail.com>
date Mon, 22 Mar 2010 13:33:29 -0400
parents
children 20ebc1f2a9fe
comparison
equal deleted inserted replaced
275:7b4507295eba 276:727ed56fad12
1 from pynnet import *
2 # use hacks also
3 from pynnet.utils import *
4
5 import numpy
6 import theano
7 import theano.tensor as T
8
9 from itertools import izip
10
11 class cdae(LayerStack):
12 def __init__(self, filter_size, num_filt, num_in, subsampling, corruption,
13 dtype, img_shape):
14 LayerStack.__init__(self, [ConvAutoencoder(filter_size=filter_size,
15 num_filt=num_filt,
16 num_in=num_in,
17 noisyness=corruption,
18 dtype=dtype,
19 image_shape=img_shape),
20 MaxPoolLayer(subsampling)])
21
22 def build(self, input):
23 LayerStack.build(self, input)
24 self.cost = self.layers[0].cost
25
26 def cdae_out_size(in_size, filt_size, num_filt, num_in, subs):
27 out = [None] * 3
28 out[0] = num_filt
29 out[1] = (in_size[1]-filt_size[0]+1)/subs[0]
30 out[2] = (in_size[2]-filt_size[1]+1)/subs[1]
31 return out
32
33 def scdae(in_size, num_in, filter_sizes, num_filts,
34 subsamplings, corruptions, dtype):
35 layers = []
36 old_nfilt = 1
37 for fsize, nfilt, subs, corr in izip(filter_sizes, num_filts,
38 subsamplings, corruptions):
39 layers.append(cdae(fsize, nfilt, old_nfilt, subs, corr, dtype,
40 (num_in, in_size[0], in_size[1], in_size[2])))
41 in_size = cdae_out_size(in_size, fsize, nfilt, old_nfilt, subs)
42 old_nfilt = nfilt
43 return LayerStack(layers), in_size
44
45 def mlp(layer_sizes, dtype):
46 layers = []
47 old_size = layer_sizes[0]
48 for size in layer_sizes[1:]:
49 layers.append(SimpleLayer(old_size, size, activation=nlins.tanh,
50 dtype=dtype))
51 old_size = size
52 return LayerStack(layers)
53
54 def scdae_net(in_size, num_in, filter_sizes, num_filts, subsamplings,
55 corruptions, layer_sizes, out_size, dtype, batch_size):
56 rl1 = ReshapeLayer((None,)+in_size)
57 ls, outs = scdae(in_size, num_in, filter_sizes, num_filts, subsamplings,
58 corruptions, dtype)
59 outs = numpy.prod(outs)
60 rl2 = ReshapeLayer((None, outs))
61 layer_sizes = [outs]+layer_sizes
62 ls2 = mlp(layer_sizes, dtype)
63 lrl = SimpleLayer(layer_sizes[-1], out_size, activation=nlins.sigmoid)
64 return NNet([rl1, ls, rl2, ls2, lrl], error=errors.nll)
65
66 def build_funcs(batch_size, img_size, filter_sizes, num_filters, subs,
67 noise, mlp_sizes, out_size, dtype, pretrain_lr, train_lr):
68
69 n = scdae_net((1,)+img_size, batch_size, filter_sizes, num_filters, subs,
70 noise, mlp_sizes, out_size, dtype, batch_size)
71 x = T.fmatrix('x')
72 y = T.ivector('y')
73
74 def pretrainfunc(net, alpha):
75 up = trainers.get_updates(net.params, net.cost, alpha)
76 return theano.function([x], net.cost, updates=up)
77
78 def trainfunc(net, alpha):
79 up = trainers.get_updates(net.params, net.cost, alpha)
80 return theano.function([x, y], net.cost, updates=up)
81
82 n.build(x, y)
83 pretrain_funcs_opt = [pretrainfunc(l, pretrain_lr) for l in n.layers[1].layers]
84 trainf_opt = trainfunc(n, train_lr)
85 evalf_opt = theano.function([x, y], errors.class_error(n.output, y))
86
87 clear_imgshape(n)
88 n.build(x, y)
89 pretrain_funcs_reg = [pretrainfunc(l, 0.01) for l in n.layers[1].layers]
90 trainf_reg = trainfunc(n, 0.1)
91 evalf_reg = theano.function([x, y], errors.class_error(n.output, y))
92
93 def select_f(f1, f2, bsize):
94 def f(x):
95 if x.shape[0] == bsize:
96 return f1(x)
97 else:
98 return f2(x)
99 return f
100
101 pretrain_funcs = [select_f(p_opt, p_reg, batch_size) for p_opt, p_reg in zip(pretrain_funcs_opt, pretrain_funcs_reg)]
102
103 def select_f2(f1, f2, bsize):
104 def f(x, y):
105 if x.shape[0] == bsize:
106 return f1(x, y)
107 else:
108 return f2(x, y)
109 return f
110
111 trainf = select_f2(trainf_opt, trainf_reg, batch_size)
112 evalf = select_f2(evalf_opt, evalf_reg, batch_size)
113 return pretrain_funcs, trainf, evalf
114
115 def do_pretrain(pretrain_funcs, pretrain_epochs):
116 for f in pretrain_funcs:
117 for i in xrange(pretrain_epochs):
118 f()
119
120 def massage_funcs(batch_size, dset, pretrain_funcs, trainf, evalf):
121 def pretrain_f(f):
122 def res():
123 for x, y in dset.train(batch_size):
124 print "pretrain:", f(x)
125 return res
126
127 pretrain_fs = map(pretrain_f, pretrain_funcs)
128
129 def train_f(f, dsetf):
130 def dset_it():
131 while True:
132 for x, y in dsetf(batch_size):
133 yield f(x, y)
134 it = dset_it()
135 return lambda: it.next()
136
137 train = train_f(trainf, dset.train)
138
139 def eval_f(f, dsetf):
140 def res():
141 c = 0
142 i = 0
143 for x, y in dsetf(batch_size):
144 i += x.shape[0]
145 c += f(x, y)*x.shape[0]
146 return c/i
147 return res
148
149 test = eval_f(evalf, dset.test)
150 valid = eval_f(evalf, dset.valid)
151
152 return pretrain_fs, train, valid, test
153
154 def run_exp(state, channel):
155 from ift6266 import datasets
156 from sgd_opt import sgd_opt
157 import sys, time
158
159 channel.save()
160
161 # params: bsize, pretrain_lr, train_lr, nfilts1, nfilts2, nftils3, nfilts4
162 # pretrain_rounds
163
164 dset = dataset.nist_all()
165
166 nfilts = []
167 if state.nfilts1 != 0:
168 nfilts.append(state.nfilts1)
169 if state.nfilts2 != 0:
170 nfilts.append(state.nfilts2)
171 if state.nfilts3 != 0:
172 nfilts.append(state.nfilts3)
173 if state.nfilts4 != 0:
174 nfilts.append(state.nfilts4)
175
176 fsizes = [(5,5)]*len(nfilts)
177 subs = [(2,2)]*len(nfilts)
178 noise = [state.noise]*len(nfilts)
179
180 pretrain_funcs, trainf, evalf = build_funcs(
181 img_size=(32, 32),
182 batch_size=state.bsize,
183 filter_sizes=fsizes,
184 num_filters=nfilts,
185 subs=subs,
186 noise=noise,
187 mlp_sizes=[state.mlp_sz],
188 out_size=62,
189 dtype=numpy.float32,
190 pretrain_lr=state.pretrain_lr,
191 train_lr=state.train_lr)
192
193 pretrain_fs, train, valid, test = massage_funcs(
194 state.bsize, dset, pretrain_funcs, trainf, evalf)
195
196 do_pretrain(pretrain_fs, state.pretrain_rounds)
197
198 sgd_opt(train, valid, test, training_epochs=100000, patience=10000,
199 patience_increase=2., improvement_threshold=0.995,
200 validation_frequency=2500)
201
202 if __name__ == '__main__':
203 from ift6266 import datasets
204 from sgd_opt import sgd_opt
205 import sys, time
206
207 batch_size = 100
208 dset = datasets.mnist(200)
209
210 pretrain_funcs, trainf, evalf = build_funcs(
211 img_size = (28, 28),
212 batch_size=batch_size, filter_sizes=[(5,5), (5,5)],
213 num_filters=[4, 3], subs=[(2,2), (2,2)], noise=[0.2, 0.2],
214 mlp_sizes=[500], out_size=10, dtype=numpy.float32,
215 pretrain_lr=0.01, train_lr=0.1)
216
217 pretrain_fs, train, valid, test = massage_funcs(
218 batch_size, dset, pretrain_funcs, trainf, evalf)
219
220 print "pretraining ...",
221 sys.stdout.flush()
222 start = time.time()
223 do_pretrain(pretrain_fs, 0)
224 end = time.time()
225 print "done (in", end-start, "s)"
226
227 sgd_opt(train, valid, test, training_epochs=1000, patience=1000,
228 patience_increase=2., improvement_threshold=0.995,
229 validation_frequency=500)