Mercurial > ift6266
annotate deep/convolutional_dae/scdae.py @ 288:80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
author | Arnaud Bergeron <abergeron@gmail.com> |
---|---|
date | Fri, 26 Mar 2010 17:24:17 -0400 |
parents | 20ebc1f2a9fe |
children | 518589bfee55 |
rev | line source |
---|---|
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
1 from pynnet import * |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
2 # use hacks also |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
3 from pynnet.utils import * |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
4 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
5 import numpy |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
6 import theano |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
7 import theano.tensor as T |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
8 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
9 from itertools import izip |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
10 from ift6266.utils.seriestables import * |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
11 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
12 class cdae(LayerStack): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
13 def __init__(self, filter_size, num_filt, num_in, subsampling, corruption, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
14 dtype, img_shape): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
15 LayerStack.__init__(self, [ConvAutoencoder(filter_size=filter_size, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
16 num_filt=num_filt, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
17 num_in=num_in, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
18 noisyness=corruption, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
19 dtype=dtype, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
20 image_shape=img_shape), |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
21 MaxPoolLayer(subsampling)]) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
22 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
23 def build(self, input): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
24 LayerStack.build(self, input) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
25 self.cost = self.layers[0].cost |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
26 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
27 def cdae_out_size(in_size, filt_size, num_filt, num_in, subs): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
28 out = [None] * 3 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
29 out[0] = num_filt |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
30 out[1] = (in_size[1]-filt_size[0]+1)/subs[0] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
31 out[2] = (in_size[2]-filt_size[1]+1)/subs[1] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
32 return out |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
33 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
34 def scdae(in_size, num_in, filter_sizes, num_filts, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
35 subsamplings, corruptions, dtype): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
36 layers = [] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
37 old_nfilt = 1 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
38 for fsize, nfilt, subs, corr in izip(filter_sizes, num_filts, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
39 subsamplings, corruptions): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
40 layers.append(cdae(fsize, nfilt, old_nfilt, subs, corr, dtype, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
41 (num_in, in_size[0], in_size[1], in_size[2]))) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
42 in_size = cdae_out_size(in_size, fsize, nfilt, old_nfilt, subs) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
43 old_nfilt = nfilt |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
44 return LayerStack(layers), in_size |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
45 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
46 def mlp(layer_sizes, dtype): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
47 layers = [] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
48 old_size = layer_sizes[0] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
49 for size in layer_sizes[1:]: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
50 layers.append(SimpleLayer(old_size, size, activation=nlins.tanh, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
51 dtype=dtype)) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
52 old_size = size |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
53 return LayerStack(layers) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
54 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
55 def scdae_net(in_size, num_in, filter_sizes, num_filts, subsamplings, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
56 corruptions, layer_sizes, out_size, dtype, batch_size): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
57 rl1 = ReshapeLayer((None,)+in_size) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
58 ls, outs = scdae(in_size, num_in, filter_sizes, num_filts, subsamplings, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
59 corruptions, dtype) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
60 outs = numpy.prod(outs) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
61 rl2 = ReshapeLayer((None, outs)) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
62 layer_sizes = [outs]+layer_sizes |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
63 ls2 = mlp(layer_sizes, dtype) |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
64 lrl = SimpleLayer(layer_sizes[-1], out_size, activation=nlins.softmax) |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
65 return NNet([rl1, ls, rl2, ls2, lrl], error=errors.nll) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
66 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
67 def build_funcs(batch_size, img_size, filter_sizes, num_filters, subs, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
68 noise, mlp_sizes, out_size, dtype, pretrain_lr, train_lr): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
69 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
70 n = scdae_net((1,)+img_size, batch_size, filter_sizes, num_filters, subs, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
71 noise, mlp_sizes, out_size, dtype, batch_size) |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
72 |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
73 n.save('start.net') |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
74 |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
75 x = T.fmatrix('x') |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
76 y = T.ivector('y') |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
77 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
78 def pretrainfunc(net, alpha): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
79 up = trainers.get_updates(net.params, net.cost, alpha) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
80 return theano.function([x], net.cost, updates=up) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
81 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
82 def trainfunc(net, alpha): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
83 up = trainers.get_updates(net.params, net.cost, alpha) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
84 return theano.function([x, y], net.cost, updates=up) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
85 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
86 n.build(x, y) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
87 pretrain_funcs_opt = [pretrainfunc(l, pretrain_lr) for l in n.layers[1].layers] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
88 trainf_opt = trainfunc(n, train_lr) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
89 evalf_opt = theano.function([x, y], errors.class_error(n.output, y)) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
90 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
91 clear_imgshape(n) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
92 n.build(x, y) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
93 pretrain_funcs_reg = [pretrainfunc(l, 0.01) for l in n.layers[1].layers] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
94 trainf_reg = trainfunc(n, 0.1) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
95 evalf_reg = theano.function([x, y], errors.class_error(n.output, y)) |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
96 |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
97 def select_f(f1, f2, bsize): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
98 def f(x): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
99 if x.shape[0] == bsize: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
100 return f1(x) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
101 else: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
102 return f2(x) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
103 return f |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
104 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
105 pretrain_funcs = [select_f(p_opt, p_reg, batch_size) for p_opt, p_reg in zip(pretrain_funcs_opt, pretrain_funcs_reg)] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
106 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
107 def select_f2(f1, f2, bsize): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
108 def f(x, y): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
109 if x.shape[0] == bsize: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
110 return f1(x, y) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
111 else: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
112 return f2(x, y) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
113 return f |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
114 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
115 trainf = select_f2(trainf_opt, trainf_reg, batch_size) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
116 evalf = select_f2(evalf_opt, evalf_reg, batch_size) |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
117 return pretrain_funcs, trainf, evalf, n |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
118 |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
119 def do_pretrain(pretrain_funcs, pretrain_epochs, serie): |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
120 for layer, f in enumerate(pretrain_funcs): |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
121 for epoch in xrange(pretrain_epochs): |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
122 serie.append((layer, epoch), f()) |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
123 |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
124 def massage_funcs(train_it, dset, batch_size, pretrain_funcs, trainf, evalf): |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
125 def pretrain_f(f): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
126 def res(): |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
127 for x, y in train_it: |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
128 yield f(x) |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
129 it = res() |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
130 return lambda: it.next() |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
131 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
132 pretrain_fs = map(pretrain_f, pretrain_funcs) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
133 |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
134 def train_f(f): |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
135 def dset_it(): |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
136 for x, y in train_it: |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
137 yield f(x, y) |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
138 it = dset_it() |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
139 return lambda: it.next() |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
140 |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
141 train = train_f(trainf) |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
142 |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
143 def eval_f(f, dsetf): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
144 def res(): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
145 c = 0 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
146 i = 0 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
147 for x, y in dsetf(batch_size): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
148 i += x.shape[0] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
149 c += f(x, y)*x.shape[0] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
150 return c/i |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
151 return res |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
152 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
153 test = eval_f(evalf, dset.test) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
154 valid = eval_f(evalf, dset.valid) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
155 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
156 return pretrain_fs, train, valid, test |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
157 |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
158 def repeat_itf(itf, *args, **kwargs): |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
159 while True: |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
160 for e in itf(*args, **kwargs): |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
161 yield e |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
162 |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
163 def create_series(): |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
164 import tables |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
165 |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
166 series = {} |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
167 h5f = tables.openFile('series.h5', 'w') |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
168 |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
169 series['recons_error'] = AccumulatorSeriesWrapper( |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
170 base_series=ErrorSeries(error_name='reconstruction_error', |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
171 table_name='reconstruction_error', |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
172 hdf5_file=h5f, |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
173 index_names=('layer', 'epoch'), |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
174 title="Reconstruction error (mse)") |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
175 reduce_every=100) |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
176 |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
177 series['training_err'] = AccumulatorSeriesWrapper( |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
178 base_series=ErrorSeries(error_name='training_error', |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
179 table_name='training_error' |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
180 hdf5_file=h5f, |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
181 index_names=('iter',), |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
182 titles='Training error (nll)') |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
183 reduce_every=100) |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
184 |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
185 series['valid_err'] = ErrorSeries(error_name='valid_error', |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
186 table_name='valid_error' |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
187 hdf5_file=h5f, |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
188 index_names=('iter',), |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
189 titles='Validation error (class)') |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
190 |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
191 series['test_err'] = ErrorSeries(error_name='test_error', |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
192 table_name='test_error' |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
193 hdf5_file=h5f, |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
194 index_names=('iter',), |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
195 titles='Test error (class)') |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
196 |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
197 def run_exp(state, channel): |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
198 from ift6266 import datasets |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
199 from sgd_opt import sgd_opt |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
200 import sys, time |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
201 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
202 # params: bsize, pretrain_lr, train_lr, nfilts1, nfilts2, nftils3, nfilts4 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
203 # pretrain_rounds |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
204 |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
205 pylearn.version.record_versions(state, [theano,ift6266,pylearn]) |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
206 # TODO: maybe record pynnet version? |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
207 channel.save() |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
208 |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
209 dset = dataset.nist_all() |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
210 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
211 nfilts = [] |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
212 if state.nfilts1 != 0: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
213 nfilts.append(state.nfilts1) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
214 if state.nfilts2 != 0: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
215 nfilts.append(state.nfilts2) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
216 if state.nfilts3 != 0: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
217 nfilts.append(state.nfilts3) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
218 if state.nfilts4 != 0: |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
219 nfilts.append(state.nfilts4) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
220 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
221 fsizes = [(5,5)]*len(nfilts) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
222 subs = [(2,2)]*len(nfilts) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
223 noise = [state.noise]*len(nfilts) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
224 |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
225 pretrain_funcs, trainf, evalf, net = build_funcs( |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
226 img_size=(32, 32), |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
227 batch_size=state.bsize, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
228 filter_sizes=fsizes, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
229 num_filters=nfilts, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
230 subs=subs, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
231 noise=noise, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
232 mlp_sizes=[state.mlp_sz], |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
233 out_size=62, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
234 dtype=numpy.float32, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
235 pretrain_lr=state.pretrain_lr, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
236 train_lr=state.train_lr) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
237 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
238 pretrain_fs, train, valid, test = massage_funcs( |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
239 state.bsize, dset, pretrain_funcs, trainf, evalf) |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
240 |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
241 series = create_series() |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
242 |
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
243 do_pretrain(pretrain_fs, state.pretrain_rounds, series['recons_error']) |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
244 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
245 sgd_opt(train, valid, test, training_epochs=100000, patience=10000, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
246 patience_increase=2., improvement_threshold=0.995, |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
247 validation_frequency=2500, series=series, net=net) |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
248 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
249 if __name__ == '__main__': |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
250 from ift6266 import datasets |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
251 from sgd_opt import sgd_opt |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
252 import sys, time |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
253 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
254 batch_size = 100 |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
255 dset = datasets.mnist() |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
256 |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
257 pretrain_funcs, trainf, evalf, net = build_funcs( |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
258 img_size = (28, 28), |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
259 batch_size=batch_size, filter_sizes=[(5,5), (3,3)], |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
260 num_filters=[4, 4], subs=[(2,2), (2,2)], noise=[0.2, 0.2], |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
261 mlp_sizes=[500], out_size=10, dtype=numpy.float32, |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
262 pretrain_lr=0.01, train_lr=0.1) |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
263 |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
264 pretrain_fs, train, valid, test = massage_funcs( |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
265 repeat_itf(dset.train, batch_size), |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
266 dset, batch_size, |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
267 pretrain_funcs, trainf, evalf) |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
268 |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
269 print "pretraining ...", |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
270 sys.stdout.flush() |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
271 start = time.time() |
288
80ee63c3e749
Add net saving (only the best model) and error saving using SeriesTable
Arnaud Bergeron <abergeron@gmail.com>
parents:
277
diff
changeset
|
272 do_pretrain(pretrain_fs, 2500, DummySeries()) |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
273 end = time.time() |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
274 print "done (in", end-start, "s)" |
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
275 |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
276 sgd_opt(train, valid, test, training_epochs=10000, patience=1000, |
276
727ed56fad12
Add reworked code for convolutional auto-encoder.
Arnaud Bergeron <abergeron@gmail.com>
parents:
diff
changeset
|
277 patience_increase=2., improvement_threshold=0.995, |
277
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
278 validation_frequency=250) |
20ebc1f2a9fe
Use softmax for the output layer and rework the dset iterator stuff.
Arnaud Bergeron <abergeron@gmail.com>
parents:
276
diff
changeset
|
279 |