Mercurial > ift6266

annotate deep/convolutional_dae/stacked_convolutional_dae.py @ 248:7e6fecabb656

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
author humel
date Tue, 16 Mar 2010 14:46:25 -0400
parents 4d109b648c31
children 1bf046c0c84a 3919c71e3091
rev   line source
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
1 import numpy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
2 import theano
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
3 import time
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
4 import sys
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
5 import theano.tensor as T
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
6 from theano.tensor.shared_randomstreams import RandomStreams
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
7 import theano.sandbox.softsign
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
8
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
9 from theano.tensor.signal import downsample
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
10 from theano.tensor.nnet import conv
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
11
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
12 sys.path.append('../../../')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
13
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
14 from ift6266 import datasets
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
15 from ift6266.baseline.log_reg.log_reg import LogisticRegression
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
16
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
17 batch_size = 100
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
18
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
19
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
20 class SigmoidalLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
21 def __init__(self, rng, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
22
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
23 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
24
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
25 W_values = numpy.asarray( rng.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
26 low = -numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
27 high = numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
28 size = (n_in, n_out)), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
29 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
30
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
31 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
32 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
33
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
34 self.output = T.tanh(T.dot(input, self.W) + self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
35 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
36
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
37 class dA_conv(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
38
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
39 def __init__(self, input, filter_shape, corruption_level = 0.1,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
40 shared_W = None, shared_b = None, image_shape = None,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
41 poolsize = (2,2)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
42
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
43 theano_rng = RandomStreams()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
44
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
45 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
46 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
47
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
48 center = theano.shared(value = 1, name="center")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
49 scale = theano.shared(value = 2, name="scale")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
50
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
51 if shared_W != None and shared_b != None :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
52 self.W = shared_W
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
53 self.b = shared_b
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
54 else:
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
55 initial_W = numpy.asarray( numpy.random.uniform(
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
56 low = -numpy.sqrt(6./(fan_in+fan_out)),
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
57 high = numpy.sqrt(6./(fan_in+fan_out)),
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
58 size = filter_shape), dtype = theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
59 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
60 self.W = theano.shared(value = initial_W, name = "W")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
61 self.b = theano.shared(value = initial_b, name = "b")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
62
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
63
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
64 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
65
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
66 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
67
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
68 self.x = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
69
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
70 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
71
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
72 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
73 image_shape=image_shape, unroll_kern=4,unroll_batch=4, border_mode='valid')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
74
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
75
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
76 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
77
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
78
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
79 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
80 filter_shape[3] ]
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
81 da_image_shape = [ image_shape[0], filter_shape[0], image_shape[2]-filter_shape[2]+1, image_shape[3]-filter_shape[3]+1 ]
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
82 initial_W_prime = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
83 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
84 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
85 size = da_filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
86 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
87
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
88 conv2_out = conv.conv2d(self.y, self.W_prime,
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
89 filter_shape = da_filter_shape,\
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
90 image_shape = da_image_shape, \
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
91 unroll_kern=4,unroll_batch=4, \
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
92 border_mode='full')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
93
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
94 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
95
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
96 scaled_x = (self.x + center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
97
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
98 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
99
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
100 self.cost = T.mean(self.L)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
101
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
102 self.params = [ self.W, self.b, self.b_prime ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
103
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
104 class LeNetConvPoolLayer(object):
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
105
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
106 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
107 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
108
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
109 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
110 self.W = theano.shared(value=W_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
111
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
112 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
113 self.b = theano.shared(value=b_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
114
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
115 conv_out = conv.conv2d(input, self.W,
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
116 filter_shape=filter_shape, image_shape=image_shape, unroll_kern=4,unroll_batch=4)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
117
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
118
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
119 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
120 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
121
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
122 W_bound = numpy.sqrt(6./(fan_in + fan_out))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
123 self.W.value = numpy.asarray(
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
124 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
125 dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
126
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
127
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
128 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
129
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
130 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
131 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
132
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
133
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
134 class SdA():
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
135 def __init__(self, input, n_ins_mlp, conv_hidden_layers_sizes,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
136 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
137 pretrain_lr, finetune_lr):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
138
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
139 self.layers = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
140 self.pretrain_functions = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
141 self.params = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
142 self.conv_n_layers = len(conv_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
143 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
144
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
145 self.x = T.matrix('x') # the data is presented as rasterized images
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
146 self.y = T.ivector('y') # the labels are presented as 1D vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
147
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
148 for i in xrange( self.conv_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
149 filter_shape=conv_hidden_layers_sizes[i][0]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
150 image_shape=conv_hidden_layers_sizes[i][1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
151 max_poolsize=conv_hidden_layers_sizes[i][2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
152
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
153 if i == 0 :
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
154 layer_input=self.x.reshape((self.x.shape[0], 1, 32, 32))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
155 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
156 layer_input=self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
157
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
158 layer = LeNetConvPoolLayer(rng, input=layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
159 image_shape=image_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
160 filter_shape=filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
161 poolsize=max_poolsize)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
162 print 'Convolutional layer', str(i+1), 'created'
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
163
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
164 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
165 self.params += layer.params
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
166
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
167 da_layer = dA_conv(corruption_level = corruption_levels[0],
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
168 input = layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
169 shared_W = layer.W, shared_b = layer.b,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
170 filter_shape = filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
171 image_shape = image_shape )
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
172
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
173 gparams = T.grad(da_layer.cost, da_layer.params)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
174
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
175 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
176 for param, gparam in zip(da_layer.params, gparams):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
177 updates[param] = param - gparam * pretrain_lr
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
178
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
179 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
180
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
181 self.pretrain_functions += [update_fn]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
182
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
183 for i in xrange( self.mlp_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
184 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
185 input_size = n_ins_mlp
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
186 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
187 input_size = mlp_hidden_layers_sizes[i-1]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
188
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
189 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
190 if len( self.layers ) == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
191 layer_input=self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
192 else :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
193 layer_input = self.layers[-1].output.flatten(2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
194 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
195 layer_input = self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
196
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
197 layer = SigmoidalLayer(rng, layer_input, input_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
198 mlp_hidden_layers_sizes[i] )
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
199
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
200 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
201 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
202
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
203 print 'MLP layer', str(i+1), 'created'
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
204
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
205 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
206 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
207 self.params += self.logLayer.params
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
208
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
209 cost = self.logLayer.negative_log_likelihood(self.y)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
210
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
211 gparams = T.grad(cost, self.params)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
212
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
213 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
214 for param,gparam in zip(self.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
215 updates[param] = param - gparam*finetune_lr
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
216
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
217 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
218
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
219 self.errors = self.logLayer.errors(self.y)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
220
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
221 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 1, \
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
222 pretrain_lr = 0.1, training_epochs = 1000, \
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
223 kernels = [ [4,5,5] , [4,3,3] ], mlp_layers=[500], \
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
224 corruption_levels = [ 0.2, 0.2, 0.2], batch_size = batch_size, \
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
225 max_pool_layers = [ [2,2] , [2,2] ], \
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
226 dataset=datasets.nist_digits):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
227
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
228
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
229 # allocate symbolic variables for the data
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
230 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
231 x = T.matrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
232 y = T.ivector('y') # the labels are presented as 1d vector of
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
233 # [int] labels
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
234
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
235 layer0_input = x.reshape((x.shape[0],1,32,32))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
236
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
237 rng = numpy.random.RandomState(1234)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
238 conv_layers=[]
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
239 init_layer = [ [ kernels[0][0],1,kernels[0][1],kernels[0][2] ],\
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
240 [ batch_size , 1, 32, 32 ],
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
241 max_pool_layers[0] ]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
242 conv_layers.append(init_layer)
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
243
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
244 conv_n_out = (32-kernels[0][2]+1)/max_pool_layers[0][0]
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
245
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
246 for i in range(1,len(kernels)):
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
247 layer = [ [ kernels[i][0],kernels[i-1][0],kernels[i][1],kernels[i][2] ],\
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
248 [ batch_size, kernels[i-1][0], conv_n_out,conv_n_out ],
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
249 max_pool_layers[i] ]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
250 conv_layers.append(layer)
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
251 conv_n_out = (conv_n_out - kernels[i][2]+1)/max_pool_layers[i][0]
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
252 print layer [1]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
253 network = SdA(input = layer0_input, n_ins_mlp = kernels[-1][0]*conv_n_out**2,
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
254 conv_hidden_layers_sizes = conv_layers,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
255 mlp_hidden_layers_sizes = mlp_layers,
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
256 corruption_levels = corruption_levels , n_out = 62,
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
257 rng = rng , pretrain_lr = pretrain_lr ,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
258 finetune_lr = learning_rate )
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
259
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
260 test_model = theano.function([network.x, network.y], network.errors)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
261
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
262 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
263 for i in xrange(len(network.layers)-len(mlp_layers)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
264 for epoch in xrange(pretraining_epochs):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
265 for x, y in dataset.train(batch_size):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
266 c = network.pretrain_functions[i](x)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
267 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch), c
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
268
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
269 patience = 10000 # look as this many examples regardless
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
270 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
271 # FOUND
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
272 improvement_threshold = 0.995 # a relative improvement of this much is
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
273
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
274 validation_frequency = patience/2
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
275
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
276 best_params = None
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
277 best_validation_loss = float('inf')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
278 test_score = 0.
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
279 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
280
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
281 done_looping = False
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
282 epoch = 0
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
283 iter = 0
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
284
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
285 while (epoch < training_epochs) and (not done_looping):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
286 epoch = epoch + 1
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
287 for x, y in dataset.train(batch_size):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
288
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
289 cost_ij = network.finetune(x, y)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
290 iter += 1
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
291
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
292 if iter % validation_frequency == 0:
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
293 validation_losses = [test_model(xv, yv) for xv, yv in dataset.valid(batch_size)]
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
294 this_validation_loss = numpy.mean(validation_losses)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
295 print('epoch %i, iter %i, validation error %f %%' % \
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
296 (epoch, iter, this_validation_loss*100.))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
297
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
298 # if we got the best validation score until now
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
299 if this_validation_loss < best_validation_loss:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
300
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
301 #improve patience if loss improvement is good enough
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
302 if this_validation_loss < best_validation_loss * \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
303 improvement_threshold :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
304 patience = max(patience, iter * patience_increase)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
305
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
306 # save best validation score and iteration number
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
307 best_validation_loss = this_validation_loss
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
308 best_iter = iter
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
309
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
310 # test it on the test set
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
311 test_losses = [test_model(xt, yt) for xt, yt in dataset.test(batch_size)]
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
312 test_score = numpy.mean(test_losses)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
313 print((' epoch %i, iter %i, test error of best '
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
314 'model %f %%') %
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
315 (epoch, iter, test_score*100.))
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
316
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
317 if patience <= iter :
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
318 done_looping = True
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
319 break
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
320
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
321 end_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
322 print(('Optimization complete with best validation score of %f %%,'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
323 'with test performance %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
324 (best_validation_loss * 100., test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
325 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
326
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
327 if __name__ == '__main__':
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
328 sgd_optimization_mnist()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
329