Mercurial > ift6266

annotate deep/convolutional_dae/stacked_convolutional_dae.py @ 222:4cfd0eb438af

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add mnist to datasets (and supporting code).
author Arnaud Bergeron <abergeron@gmail.com>
date Thu, 11 Mar 2010 14:41:31 -0500
parents 334d2444000d
children 4d109b648c31
rev   line source
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
1 import numpy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
2 import theano
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
3 import time
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
4 import theano.tensor as T
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
5 from theano.tensor.shared_randomstreams import RandomStreams
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
6 import theano.sandbox.softsign
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
7
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
8 from theano.tensor.signal import downsample
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
9 from theano.tensor.nnet import conv
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
10
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
11 from ift6266 import datasets
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
12
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
13 from ift6266.baseline.log_reg.log_reg import LogisticRegression
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
14
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
15 class SigmoidalLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
16 def __init__(self, rng, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
17
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
18 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
19
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
20 W_values = numpy.asarray( rng.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
21 low = -numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
22 high = numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
23 size = (n_in, n_out)), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
24 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
25
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
26 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
27 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
28
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
29 self.output = T.tanh(T.dot(input, self.W) + self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
30 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
31
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
32 class dA_conv(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
33
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
34 def __init__(self, input, filter_shape, corruption_level = 0.1,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
35 shared_W = None, shared_b = None, image_shape = None,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
36 poolsize = (2,2)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
37
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
38 theano_rng = RandomStreams()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
39
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
40 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
41 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
42
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
43 center = theano.shared(value = 1, name="center")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
44 scale = theano.shared(value = 2, name="scale")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
45
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
46 if shared_W != None and shared_b != None :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
47 self.W = shared_W
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
48 self.b = shared_b
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
49 else:
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
50 initial_W = numpy.asarray( numpy.random.uniform(
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
51 low = -numpy.sqrt(6./(fan_in+fan_out)),
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
52 high = numpy.sqrt(6./(fan_in+fan_out)),
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
53 size = filter_shape), dtype = theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
54 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
55 self.W = theano.shared(value = initial_W, name = "W")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
56 self.b = theano.shared(value = initial_b, name = "b")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
57
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
58
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
59 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
60
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
61 self.W_prime=T.dtensor4('W_prime')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
62
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
63 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
64
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
65 self.x = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
66
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
67 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
68
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
69 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
70 image_shape=image_shape, border_mode='valid')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
71
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
72
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
73 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
74
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
75
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
76 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
77 filter_shape[3] ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
78 initial_W_prime = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
79 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
80 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
81 size = da_filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
82 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
83
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
84 conv2_out = conv.conv2d(self.y, self.W_prime,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
85 filter_shape = da_filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
86 border_mode='full')
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
87
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
88 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
89
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
90 scaled_x = (self.x + center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
91
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
92 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
93
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
94 self.cost = T.mean(self.L)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
95
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
96 self.params = [ self.W, self.b, self.b_prime ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
97
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
98 class LeNetConvPoolLayer(object):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
99 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
100 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
101
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
102 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
103 self.W = theano.shared(value=W_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
104
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
105 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
106 self.b = theano.shared(value=b_values)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
107
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
108 conv_out = conv.conv2d(input, self.W,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
109 filter_shape=filter_shape, image_shape=image_shape)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
110
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
111
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
112 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
113 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
114
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
115 W_bound = numpy.sqrt(6./(fan_in + fan_out))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
116 self.W.value = numpy.asarray(
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
117 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
118 dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
119
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
120
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
121 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
122
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
123 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
124 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
125
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
126
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
127 class SdA():
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
128 def __init__(self, input, n_ins_mlp, conv_hidden_layers_sizes,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
129 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
130 pretrain_lr, finetune_lr):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
131
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
132 self.layers = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
133 self.pretrain_functions = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
134 self.params = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
135 self.conv_n_layers = len(conv_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
136 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
137
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
138 self.x = T.matrix('x') # the data is presented as rasterized images
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
139 self.y = T.ivector('y') # the labels are presented as 1D vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
140
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
141 for i in xrange( self.conv_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
142 filter_shape=conv_hidden_layers_sizes[i][0]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
143 image_shape=conv_hidden_layers_sizes[i][1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
144 max_poolsize=conv_hidden_layers_sizes[i][2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
145
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
146 if i == 0 :
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
147 layer_input=self.x.reshape((self.x.shape[0], 1, 32, 32))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
148 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
149 layer_input=self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
150
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
151 layer = LeNetConvPoolLayer(rng, input=layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
152 image_shape=image_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
153 filter_shape=filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
154 poolsize=max_poolsize)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
155 print 'Convolutional layer', str(i+1), 'created'
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
156
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
157 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
158 self.params += layer.params
215
334d2444000d Changes that enable using this code when floatX=float32
Dumitru Erhan <dumitru.erhan@gmail.com>
parents: 200
diff changeset
159
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
160 da_layer = dA_conv(corruption_level = corruption_levels[0],
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
161 input = layer_input,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
162 shared_W = layer.W, shared_b = layer.b,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
163 filter_shape = filter_shape,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
164 image_shape = image_shape )
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
165
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
166 gparams = T.grad(da_layer.cost, da_layer.params)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
167
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
168 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
169 for param, gparam in zip(da_layer.params, gparams):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
170 updates[param] = param - gparam * pretrain_lr
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
171
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
172 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
173
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
174 self.pretrain_functions += [update_fn]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
175
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
176 for i in xrange( self.mlp_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
177 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
178 input_size = n_ins_mlp
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
179 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
180 input_size = mlp_hidden_layers_sizes[i-1]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
181
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
182 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
183 if len( self.layers ) == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
184 layer_input=self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
185 else :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
186 layer_input = self.layers[-1].output.flatten(2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
187 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
188 layer_input = self.layers[-1].output
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
189
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
190 layer = SigmoidalLayer(rng, layer_input, input_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
191 mlp_hidden_layers_sizes[i] )
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
192
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
193 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
194 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
195
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
196 print 'MLP layer', str(i+1), 'created'
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
197
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
198 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
199 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
200 self.params += self.logLayer.params
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
201
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
202 cost = self.logLayer.negative_log_likelihood(self.y)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
203
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
204 gparams = T.grad(cost, self.params)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
205
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
206 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
207 for param,gparam in zip(self.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
208 updates[param] = param - gparam*finetune_lr
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
209
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
210 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
211
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
212 self.errors = self.logLayer.errors(self.y)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
213
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
214 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 2, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
215 pretrain_lr = 0.01, training_epochs = 1000, \
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
216 dataset=datasets.nist_digits):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
217
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
218 batch_size = 500 # size of the minibatch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
219
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
220 # allocate symbolic variables for the data
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
221 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
222 x = T.matrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
223 y = T.ivector('y') # the labels are presented as 1d vector of
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
224 # [int] labels
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
225 layer0_input = x.reshape((x.shape[0],1,32,32))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
226
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
227
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
228 # Setup the convolutional layers with their DAs(add as many as you want)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
229 corruption_levels = [ 0.2, 0.2, 0.2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
230 rng = numpy.random.RandomState(1234)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
231 ker1=2
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
232 ker2=2
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
233 conv_layers=[]
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
234 conv_layers.append([[ker1,1,5,5], None, [2,2] ])
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
235 conv_layers.append([[ker2,ker1,5,5], None, [2,2] ])
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
236
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
237 # Setup the MLP layers of the network
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
238 mlp_layers=[500]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
239
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
240 network = SdA(input = layer0_input, n_ins_mlp = ker2*4*4,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
241 conv_hidden_layers_sizes = conv_layers,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
242 mlp_hidden_layers_sizes = mlp_layers,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
243 corruption_levels = corruption_levels , n_out = 10,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
244 rng = rng , pretrain_lr = pretrain_lr ,
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
245 finetune_lr = learning_rate )
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
246
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
247 test_model = theano.function([network.x, network.y], network.errors)
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
248
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
249 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
250 for i in xrange(len(network.layers)-len(mlp_layers)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
251 for epoch in xrange(pretraining_epochs):
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
252 for x, y in dataset.train(batch_size):
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
253 c = network.pretrain_functions[i](x)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
254 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch), c
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
255
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
256 patience = 10000 # look as this many examples regardless
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
257 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
258 # FOUND
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
259 improvement_threshold = 0.995 # a relative improvement of this much is
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
260
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
261 validation_frequency = patience/2
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
262
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
263 best_params = None
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
264 best_validation_loss = float('inf')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
265 test_score = 0.
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
266 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
267
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
268 done_looping = False
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
269 epoch = 0
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
270 iter = 0
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
271
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
272 while (epoch < training_epochs) and (not done_looping):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
273 epoch = epoch + 1
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
274 for x, y in dataset.train(batch_size):
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
275
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
276 cost_ij = network.finetune(x, y)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
277 iter += 1
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
278
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
279 if iter % validation_frequency == 0:
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
280 validation_losses = [test_model(xv, yv) for xv, yv in dataset.valid(batch_size)]
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
281 this_validation_loss = numpy.mean(validation_losses)
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
282 print('epoch %i, iter %i, validation error %f %%' % \
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
283 (epoch, iter, this_validation_loss*100.))
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
284
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
285 # if we got the best validation score until now
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
286 if this_validation_loss < best_validation_loss:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
287
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
288 #improve patience if loss improvement is good enough
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
289 if this_validation_loss < best_validation_loss * \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
290 improvement_threshold :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
291 patience = max(patience, iter * patience_increase)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
292
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
293 # save best validation score and iteration number
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
294 best_validation_loss = this_validation_loss
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
295 best_iter = iter
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
296
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
297 # test it on the test set
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
298 test_losses = [test_model(xt, yt) for xt, yt in dataset.test(batch_size)]
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
299 test_score = numpy.mean(test_losses)
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
300 print((' epoch %i, iter %i, test error of best '
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
301 'model %f %%') %
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
302 (epoch, iter, test_score*100.))
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
303
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
304 if patience <= iter :
200
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
305 done_looping = True
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
306 break
3f2cc90ad51c Adapt the sdae code for ift6266.datasets input.
Arnaud Bergeron <abergeron@gmail.com>
parents: 167
diff changeset
307
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
308 end_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
309 print(('Optimization complete with best validation score of %f %%,'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
310 'with test performance %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
311 (best_validation_loss * 100., test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
312 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
313
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
314 if __name__ == '__main__':
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
315 sgd_optimization_mnist()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
316