Mercurial > ift6266

annotate deep/convolutional_dae/stacked_convolutional_dae.py @ 266:1e4e60ddadb1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge. Ah, et dans le dernier commit, j'avais oublié de mentionner que j'ai ajouté du code pour gérer l'isolation de différents clones pour rouler des expériences et modifier le code en même temps.
author fsavard
date Fri, 19 Mar 2010 10:56:16 -0400
parents 0c0f0b3f6a93
children
rev   line source
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
1 import numpy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
2 import theano
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
3 import time
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
4 import sys
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
5 import theano.tensor as T
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
6 from theano.tensor.shared_randomstreams import RandomStreams
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
7 #import theano.sandbox.softsign
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
8
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
9 from theano.tensor.signal import downsample
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
10 from theano.tensor.nnet import conv
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
11
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
12 from ift6266 import datasets
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
13 from ift6266.baseline.log_reg.log_reg import LogisticRegression
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
14
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
15 batch_size = 100
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
16
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
17 class SigmoidalLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
18 def __init__(self, rng, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
19
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
20 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
21
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
22 W_values = numpy.asarray( rng.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
23 low = -numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
24 high = numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
25 size = (n_in, n_out)), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
26 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
27
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
28 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
29 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
30
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
31 self.output = T.tanh(T.dot(input, self.W) + self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
32 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
33
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
34 class dA_conv(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
35
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
36 def __init__(self, input, filter_shape, corruption_level = 0.1,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
37 shared_W = None, shared_b = None, image_shape = None,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
38 poolsize = (2,2)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
39
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
40 theano_rng = RandomStreams()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
41
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
42 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
43 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
44
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
45 center = theano.shared(value = 1, name="center")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
46 scale = theano.shared(value = 2, name="scale")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
47
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
48 if shared_W != None and shared_b != None :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
49 self.W = shared_W
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
50 self.b = shared_b
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
51 else:
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
52 initial_W = numpy.asarray( numpy.random.uniform(
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
53 low = -numpy.sqrt(6./(fan_in+fan_out)),
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
54 high = numpy.sqrt(6./(fan_in+fan_out)),
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
55 size = filter_shape), dtype = theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
56 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
57 self.W = theano.shared(value = initial_W, name = "W")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
58 self.b = theano.shared(value = initial_b, name = "b")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
59
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
60
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
61 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
62
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
63 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
64
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
65 self.x = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
66
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
67 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
68
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
69 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
70 image_shape=image_shape, border_mode='valid')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
71
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
72 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
73
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
74 da_filter_shape = [ filter_shape[1], filter_shape[0],
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
75 filter_shape[2], filter_shape[3] ]
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
76 initial_W_prime = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
77 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
78 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
79 size = da_filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
80 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
81
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
82 conv2_out = conv.conv2d(self.y, self.W_prime,
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
83 filter_shape = da_filter_shape,
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
84 border_mode='full')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
85
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
86 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
87
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
88 scaled_x = (self.x + center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
89
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
90 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
91
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
92 self.cost = T.mean(self.L)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
93
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
94 self.params = [ self.W, self.b, self.b_prime ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
95
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
96 class LeNetConvPoolLayer(object):
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
97
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
98 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
99 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
100
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
101 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
102 self.W = theano.shared(value=W_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
103
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
104 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
105 self.b = theano.shared(value=b_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
106
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
107 conv_out = conv.conv2d(input, self.W,
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
108 filter_shape=filter_shape, image_shape=image_shape)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
109
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
110
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
111 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
112 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
113
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
114 W_bound = numpy.sqrt(6./(fan_in + fan_out))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
115 self.W.value = numpy.asarray(
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
116 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
117 dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
118
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
119
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
120 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
121
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
122 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
123 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
124
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
125
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
126 class SdA():
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
127 def __init__(self, input, n_ins_mlp, conv_hidden_layers_sizes,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
128 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
129 pretrain_lr, finetune_lr, img_shape):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
130
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
131 self.layers = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
132 self.pretrain_functions = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
133 self.params = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
134 self.conv_n_layers = len(conv_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
135 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
136
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
137 self.x = T.matrix('x') # the data is presented as rasterized images
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
138 self.y = T.ivector('y') # the labels are presented as 1D vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
139
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
140 for i in xrange( self.conv_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
141 filter_shape=conv_hidden_layers_sizes[i][0]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
142 image_shape=conv_hidden_layers_sizes[i][1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
143 max_poolsize=conv_hidden_layers_sizes[i][2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
144
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
145 if i == 0 :
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
146 layer_input=self.x.reshape((self.x.shape[0], 1) + img_shape)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
147 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
148 layer_input=self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
149
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
150 layer = LeNetConvPoolLayer(rng, input=layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
151 image_shape=image_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
152 filter_shape=filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
153 poolsize=max_poolsize)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
154 print 'Convolutional layer', str(i+1), 'created'
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
155
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
156 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
157 self.params += layer.params
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
158
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
159 da_layer = dA_conv(corruption_level = corruption_levels[0],
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
160 input = layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
161 shared_W = layer.W, shared_b = layer.b,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
162 filter_shape = filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
163 image_shape = image_shape )
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
164
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
165 gparams = T.grad(da_layer.cost, da_layer.params)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
166
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
167 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
168 for param, gparam in zip(da_layer.params, gparams):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
169 updates[param] = param - gparam * pretrain_lr
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
170
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
171 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
172
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
173 self.pretrain_functions += [update_fn]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
174
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
175 for i in xrange( self.mlp_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
176 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
177 input_size = n_ins_mlp
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
178 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
179 input_size = mlp_hidden_layers_sizes[i-1]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
180
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
181 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
182 if len( self.layers ) == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
183 layer_input=self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
184 else :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
185 layer_input = self.layers[-1].output.flatten(2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
186 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
187 layer_input = self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
188
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
189 layer = SigmoidalLayer(rng, layer_input, input_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
190 mlp_hidden_layers_sizes[i] )
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
191
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
192 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
193 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
194
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
195 print 'MLP layer', str(i+1), 'created'
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
196
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
197 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
198 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
199 self.params += self.logLayer.params
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
200
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
201 cost = self.logLayer.negative_log_likelihood(self.y)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
202
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
203 gparams = T.grad(cost, self.params)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
204
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
205 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
206 for param,gparam in zip(self.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
207 updates[param] = param - gparam*finetune_lr
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
208
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
209 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
210
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
211 self.errors = self.logLayer.errors(self.y)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
212
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
213 def sgd_optimization_mnist(learning_rate=0.1, pretraining_epochs = 1,
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
214 pretrain_lr = 0.1, training_epochs = 1000,
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
215 kernels = [[4,5,5], [4,3,3]], mlp_layers=[500],
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
216 corruption_levels = [0.2, 0.2, 0.2],
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
217 batch_size = batch_size, img_shape=(28, 28),
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
218 max_pool_layers = [[2,2], [2,2]],
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
219 dataset=datasets.mnist(5000)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
220
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
221 # allocate symbolic variables for the data
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
222 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
223 x = T.matrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
224 y = T.ivector('y') # the labels are presented as 1d vector of
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
225 # [int] labels
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
226
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
227 layer0_input = x.reshape((x.shape[0],1)+img_shape)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
228
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
229 rng = numpy.random.RandomState(1234)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
230 conv_layers=[]
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
231 init_layer = [[kernels[0][0],1,kernels[0][1],kernels[0][2]],
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
232 None, # do not specify the batch size since it can
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
233 # change for the last one and then theano will
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
234 # crash.
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
235 max_pool_layers[0]]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
236 conv_layers.append(init_layer)
248
7e6fecabb656 Optimized the call of ConvOp by specifying additional parameters. Specified image shape of the da_conv layer.
humel
parents: 247
diff changeset
237
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
238 conv_n_out = (img_shape[0]-kernels[0][2]+1)/max_pool_layers[0][0]
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
239
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
240 for i in range(1,len(kernels)):
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
241 layer = [[kernels[i][0],kernels[i-1][0],kernels[i][1],kernels[i][2]],
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
242 None, # same comment as for init_layer
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
243 max_pool_layers[i] ]
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
244 conv_layers.append(layer)
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
245 conv_n_out = (conv_n_out - kernels[i][2]+1)/max_pool_layers[i][0]
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
246
247
4d109b648c31 Fixed dataset import. Removed unuseful code from da_conv. Keys parameters are now passed as arguments.
humel
parents: 215
diff changeset
247 network = SdA(input = layer0_input, n_ins_mlp = kernels[-1][0]*conv_n_out**2,
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
248 conv_hidden_layers_sizes = conv_layers,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
249 mlp_hidden_layers_sizes = mlp_layers,
259
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
250 corruption_levels = corruption_levels, n_out = 62,
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
251 rng = rng , pretrain_lr = pretrain_lr,
3919c71e3091 Make img_size a parameter, and remove the passing of the image size to the ConvOp. This will have to get back in later somehow.
Arnaud Bergeron <abergeron@gmail.com>
parents: 248
diff changeset
252 finetune_lr = learning_rate, img_shape=img_shape)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
253
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
254 test_model = theano.function([network.x, network.y], network.errors)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
255
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
256 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
257 for i in xrange(len(network.layers)-len(mlp_layers)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
258 for epoch in xrange(pretraining_epochs):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
259 for x, y in dataset.train(batch_size):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
260 c = network.pretrain_functions[i](x)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
261 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch), c
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
262
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
263 patience = 10000 # look as this many examples regardless
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
264 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
265 # FOUND
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
266 improvement_threshold = 0.995 # a relative improvement of this much is
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
267
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
268 validation_frequency = patience/2
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
269
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
270 best_params = None
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
271 best_validation_loss = float('inf')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
272 test_score = 0.
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
273 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
274
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
275 done_looping = False
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
276 epoch = 0
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
277 iter = 0
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
278
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
279 while (epoch < training_epochs) and (not done_looping):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
280 epoch = epoch + 1
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
281 for x, y in dataset.train(batch_size):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
282
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
283 cost_ij = network.finetune(x, y)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
284 iter += 1
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
285
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
286 if iter % validation_frequency == 0:
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
287 validation_losses = [test_model(xv, yv) for xv, yv in dataset.valid(batch_size)]
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
288 this_validation_loss = numpy.mean(validation_losses)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
289 print('epoch %i, iter %i, validation error %f %%' % \
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
290 (epoch, iter, this_validation_loss*100.))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
291
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
292 # if we got the best validation score until now
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
293 if this_validation_loss < best_validation_loss:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
294
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
295 #improve patience if loss improvement is good enough
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
296 if this_validation_loss < best_validation_loss * \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
297 improvement_threshold :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
298 patience = max(patience, iter * patience_increase)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
299
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
300 # save best validation score and iteration number
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
301 best_validation_loss = this_validation_loss
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
302 best_iter = iter
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
303
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
304 # test it on the test set
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
305 test_losses = [test_model(xt, yt) for xt, yt in dataset.test(batch_size)]
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
306 test_score = numpy.mean(test_losses)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
307 print((' epoch %i, iter %i, test error of best '
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
308 'model %f %%') %
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
309 (epoch, iter, test_score*100.))
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
310
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
311 if patience <= iter :
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
312 done_looping = True
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
313 break
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
314
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
315 end_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
316 print(('Optimization complete with best validation score of %f %%,'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
317 'with test performance %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
318 (best_validation_loss * 100., test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
319 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
320
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
321 if __name__ == '__main__':
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
322 sgd_optimization_mnist()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
323