Mercurial > ift6266

annotate deep/convolutional_dae/stacked_convolutional_dae.py @ 261:6d16a2bf142b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
important bug fix in pipeline corresponding to an iterator always giving 0 complexity, need to redo 10% of the P07 dataset
author Xavier Glorot <glorotxa@iro.umontreal.ca>
date Wed, 17 Mar 2010 16:41:16 -0400
parents 1bf046c0c84a
children 0c0f0b3f6a93
rev   line source
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
1 import numpy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
2 import theano
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
3 import time
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
4 import sys
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
5 import theano.tensor as T
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
6 from theano.tensor.shared_randomstreams import RandomStreams
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
7 import theano.sandbox.softsign
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
8
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
9 from theano.tensor.signal import downsample
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
10 from theano.tensor.nnet import conv
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
11
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
12 sys.path.append('../../../')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
13
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
14 from ift6266 import datasets
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
15 from ift6266.baseline.log_reg.log_reg import LogisticRegression
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
16
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
17 batch_size = 100
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
18
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
19
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
20 class SigmoidalLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
21 def __init__(self, rng, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
22
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
23 self.input = input
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
24
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
25 W_values = numpy.asarray( rng.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
26 low = -numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
27 high = numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
28 size = (n_in, n_out)), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
29 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
30
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
31 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
32 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
33
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
34 self.output = T.tanh(T.dot(input, self.W) + self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
35 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
36
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
37 class dA_conv(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
38
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
39 def __init__(self, input, filter_shape, corruption_level = 0.1,
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
40 shared_W = None, shared_b = None, image_shape = None):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
41
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
42 theano_rng = RandomStreams()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
43
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
44 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
45 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
46
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
47 center = theano.shared(value = 1, name="center")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
48 scale = theano.shared(value = 2, name="scale")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
49
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
50 if shared_W != None and shared_b != None :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
51 self.W = shared_W
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
52 self.b = shared_b
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
53 else:
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
54 initial_W = numpy.asarray( numpy.random.uniform(
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
55 low = -numpy.sqrt(6./(fan_in+fan_out)),
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
56 high = numpy.sqrt(6./(fan_in+fan_out)),
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
57 size = filter_shape), dtype = theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
58 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
59 self.W = theano.shared(value = initial_W, name = "W")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
60 self.b = theano.shared(value = initial_b, name = "b")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
61
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
62
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
63 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
64
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
65 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
66
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
67 self.x = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
68
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
69 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
70
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
71 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
72 image_shape=image_shape,
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
73 unroll_kern=4,unroll_batch=4,
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
74 border_mode='valid')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
75
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
76
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
77 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
78
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
79
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
80 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
81 filter_shape[3] ]
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
82 da_image_shape = [ batch_size, filter_shape[0], image_shape[2]-filter_shape[2]+1, image_shape[3]-filter_shape[3]+1 ]
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
83 #import pdb; pdb.set_trace()
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
84 initial_W_prime = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
85 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
86 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
87 size = da_filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
88 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
89
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
90 conv2_out = conv.conv2d(self.y, self.W_prime,
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
91 filter_shape = da_filter_shape,\
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
92 image_shape = da_image_shape, \
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
93 unroll_kern=4,unroll_batch=4, \
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
94 border_mode='full')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
95
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
96 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
97
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
98 scaled_x = (self.x + center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
99
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
100 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
101
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
102 self.cost = T.mean(self.L)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
103
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
104 self.params = [ self.W, self.b, self.b_prime ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
105
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
106 class LeNetConvPoolLayer(object):
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
107
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
108 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
109 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
110
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
111 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
112 self.W = theano.shared(value=W_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
113
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
114 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
115 self.b = theano.shared(value=b_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
116
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
117 conv_out = conv.conv2d(input, self.W,
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
118 filter_shape=filter_shape, image_shape=image_shape,
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
119 unroll_kern=4,unroll_batch=4)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
120
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
121
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
122 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
123 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
124
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
125 W_bound = numpy.sqrt(6./(fan_in + fan_out))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
126 self.W.value = numpy.asarray(
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
127 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
128 dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
129
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
130
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
131 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
132
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
133 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
134 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
135
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
136
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
137 class SdA():
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
138 def __init__(self, input, n_ins_mlp, conv_hidden_layers_sizes,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
139 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
140 pretrain_lr, finetune_lr):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
141
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
142 self.layers = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
143 self.pretrain_functions = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
144 self.params = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
145 self.conv_n_layers = len(conv_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
146 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
147
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
148 self.x = T.matrix('x') # the data is presented as rasterized images
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
149 self.y = T.ivector('y') # the labels are presented as 1D vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
150
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
151 for i in xrange( self.conv_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
152 filter_shape=conv_hidden_layers_sizes[i][0]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
153 image_shape=conv_hidden_layers_sizes[i][1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
154 max_poolsize=conv_hidden_layers_sizes[i][2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
155
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
156 if i == 0 :
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
157 layer_input=self.x.reshape((batch_size, 1, 32, 32))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
158 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
159 layer_input=self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
160
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
161 layer = LeNetConvPoolLayer(rng, input=layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
162 image_shape=image_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
163 filter_shape=filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
164 poolsize=max_poolsize)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
165 print 'Convolutional layer', str(i+1), 'created'
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
166
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
167 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
168 self.params += layer.params
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
169
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
170 da_layer = dA_conv(corruption_level = corruption_levels[0],
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
171 input = layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
172 shared_W = layer.W, shared_b = layer.b,
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
173 filter_shape=filter_shape,
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
174 image_shape = image_shape )
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
175
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
176 gparams = T.grad(da_layer.cost, da_layer.params)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
177
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
178 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
179 for param, gparam in zip(da_layer.params, gparams):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
180 updates[param] = param - gparam * pretrain_lr
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
181
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
182 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
183
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
184 self.pretrain_functions += [update_fn]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
185
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
186 for i in xrange( self.mlp_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
187 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
188 input_size = n_ins_mlp
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
189 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
190 input_size = mlp_hidden_layers_sizes[i-1]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
191
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
192 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
193 if len( self.layers ) == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
194 layer_input=self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
195 else :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
196 layer_input = self.layers[-1].output.flatten(2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
197 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
198 layer_input = self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
199
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
200 layer = SigmoidalLayer(rng, layer_input, input_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
201 mlp_hidden_layers_sizes[i] )
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
202
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
203 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
204 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
205
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
206 print 'MLP layer', str(i+1), 'created'
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
207
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
208 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
209 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
210 self.params += self.logLayer.params
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
211
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
212 cost = self.logLayer.negative_log_likelihood(self.y)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
213
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
214 gparams = T.grad(cost, self.params)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
215
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
216 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
217 for param,gparam in zip(self.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
218 updates[param] = param - gparam*finetune_lr
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
219
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
220 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
221
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
222 self.errors = self.logLayer.errors(self.y)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
223
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
224 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 0, \
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
225 pretrain_lr = 0.1, training_epochs = 1000, \
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
226 kernels = [ [2,5,5] , [2,3,3] ], mlp_layers=[500], \
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
227 corruption_levels = [ 0.2, 0.2, 0.2], batch_size = batch_size, \
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
228 max_pool_layers = [ [2,2] , [2,2] ], \
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
229 dataset=datasets.nist_digits):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
230
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
231
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
232 # allocate symbolic variables for the data
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
233 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
234 x = T.matrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
235 y = T.ivector('y') # the labels are presented as 1d vector of
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
236 # [int] labels
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
237
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
238 layer0_input = x.reshape((batch_size,1,32,32))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
239
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
240 rng = numpy.random.RandomState(1234)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
241 conv_layers=[]
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
242 init_layer = [ [ kernels[0][0],1,kernels[0][1],kernels[0][2] ],\
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
243 [ batch_size , 1, 32, 32 ],
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
244 max_pool_layers[0] ]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
245 conv_layers.append(init_layer)
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
246
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
247 conv_n_out = int((32-kernels[0][2]+1)/max_pool_layers[0][0])
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
248 print init_layer[1]
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
249
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
250 for i in range(1,len(kernels)):
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
251 layer = [ [ kernels[i][0],kernels[i-1][0],kernels[i][1],kernels[i][2] ],\
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
252 [ batch_size, kernels[i-1][0],conv_n_out,conv_n_out ],
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
253 max_pool_layers[i] ]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
254 conv_layers.append(layer)
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
255 conv_n_out = int( (conv_n_out - kernels[i][2]+1)/max_pool_layers[i][0])
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
256 print layer [1]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
257 network = SdA(input = layer0_input, n_ins_mlp = kernels[-1][0]*conv_n_out**2,
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
258 conv_hidden_layers_sizes = conv_layers,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
259 mlp_hidden_layers_sizes = mlp_layers,
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
260 corruption_levels = corruption_levels , n_out = 62,
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
261 rng = rng , pretrain_lr = pretrain_lr ,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
262 finetune_lr = learning_rate )
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
263
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
264 test_model = theano.function([network.x, network.y], network.errors)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
265
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
266 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
267 for i in xrange(len(network.layers)-len(mlp_layers)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
268 for epoch in xrange(pretraining_epochs):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
269 for x, y in dataset.train(batch_size):
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
270 if x.shape[0] == batch_size:
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
271 c = network.pretrain_functions[i](x)
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
272
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
273 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch), c
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
274
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
275 patience = 10000 # look as this many examples regardless
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
276 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
277 # FOUND
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
278 improvement_threshold = 0.995 # a relative improvement of this much is
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
279
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
280 validation_frequency = patience/2
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
281
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
282 best_params = None
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
283 best_validation_loss = float('inf')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
284 test_score = 0.
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
285 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
286
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
287 done_looping = False
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
288 epoch = 0
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
289 iter = 0
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
290
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
291 while (epoch < training_epochs) and (not done_looping):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
292 epoch = epoch + 1
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
293 for x, y in dataset.train(batch_size):
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
294 if x.shape[0] != batch_size:
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
295 continue
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
296 cost_ij = network.finetune(x, y)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
297 iter += 1
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
298
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
299 if iter % validation_frequency == 0:
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
300 validation_losses = []
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
301 for xv, yv in dataset.valid(batch_size):
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
302 if xv.shape[0] == batch_size:
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
303 validation_losses.append(test_model(xv, yv))
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
304 this_validation_loss = numpy.mean(validation_losses)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
305 print('epoch %i, iter %i, validation error %f %%' % \
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
306 (epoch, iter, this_validation_loss*100.))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
307
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
308 # if we got the best validation score until now
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
309 if this_validation_loss < best_validation_loss:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
310
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
311 #improve patience if loss improvement is good enough
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
312 if this_validation_loss < best_validation_loss * \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
313 improvement_threshold :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
314 patience = max(patience, iter * patience_increase)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
315
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
316 # save best validation score and iteration number
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
317 best_validation_loss = this_validation_loss
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
318 best_iter = iter
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
319
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
320 # test it on the test set
249
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
321 test_losses=[]
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
322 for xt, yt in dataset.test(batch_size):
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
323 if xt.shape[0] == batch_size:
1bf046c0c84a Fixed a bug with image_shape.
humel
parents: 248
diff changeset
324 test_losses.append(test_model(xt, yt))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
325 test_score = numpy.mean(test_losses)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
326 print((' epoch %i, iter %i, test error of best '
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
327 'model %f %%') %
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
328 (epoch, iter, test_score*100.))
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
329
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
330 if patience <= iter :
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
331 done_looping = True
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
332 break
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
333
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
334 end_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
335 print(('Optimization complete with best validation score of %f %%,'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
336 'with test performance %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
337 (best_validation_loss * 100., test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
338 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
339
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
340 if __name__ == '__main__':
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
341 sgd_optimization_mnist()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
342