Mercurial > ift6266

annotate scripts/stacked_dae/stacked_convolutional_dae.py @ 156:6f3b866c0182

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
On peut maintenant launcher le pipeline avec un seed donné, résultats déterministes
author boulanni <nicolas_boulanger@hotmail.com>
date Wed, 24 Feb 2010 19:12:01 -0500
parents 128507ac4edf
children
rev   line source
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
1 import numpy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
2 import theano
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
3 import time
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
4 import theano.tensor as T
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
5 from theano.tensor.shared_randomstreams import RandomStreams
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
6 import theano.sandbox.softsign
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
7
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
8 from theano.tensor.signal import downsample
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
9 from theano.tensor.nnet import conv
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
10 import gzip
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
11 import cPickle
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
12
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
13
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
14 class LogisticRegression(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
15
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
16 def __init__(self, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
17
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
18 self.W = theano.shared( value=numpy.zeros((n_in,n_out),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
19 dtype = theano.config.floatX) )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
20
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
21 self.b = theano.shared( value=numpy.zeros((n_out,),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
22 dtype = theano.config.floatX) )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
23
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
24 self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W)+self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
25
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
26
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
27 self.y_pred=T.argmax(self.p_y_given_x, axis=1)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
28
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
29 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
30
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
31 def negative_log_likelihood(self, y):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
32 return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
33
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
34 def MSE(self, y):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
35 return -T.mean(abs((self.p_y_given_x)[T.arange(y.shape[0]),y]-y)**2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
36
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
37 def errors(self, y):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
38 if y.ndim != self.y_pred.ndim:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
39 raise TypeError('y should have the same shape as self.y_pred',
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
40 ('y', target.type, 'y_pred', self.y_pred.type))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
41
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
42
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
43 if y.dtype.startswith('int'):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
44 return T.mean(T.neq(self.y_pred, y))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
45 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
46 raise NotImplementedError()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
47
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
48
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
49 class SigmoidalLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
50 def __init__(self, rng, input, n_in, n_out):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
51
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
52 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
53
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
54 W_values = numpy.asarray( rng.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
55 low = -numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
56 high = numpy.sqrt(6./(n_in+n_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
57 size = (n_in, n_out)), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
58 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
59
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
60 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
61 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
62
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
63 self.output = T.tanh(T.dot(input, self.W) + self.b)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
64 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
65
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
66 class dA_conv(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
67
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
68 def __init__(self, corruption_level = 0.1, input = None, shared_W = None,\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
69 shared_b = None, filter_shape = None, image_shape = None, poolsize = (2,2)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
70
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
71 theano_rng = RandomStreams()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
72
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
73 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
74 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
75
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
76 center = theano.shared(value = 1, name="center")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
77 scale = theano.shared(value = 2, name="scale")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
78
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
79 if shared_W != None and shared_b != None :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
80 self.W = shared_W
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
81 self.b = shared_b
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
82 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
83 initial_W = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
84 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
85 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
86 size = filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
87 initial_b = numpy.zeros((filter_shape[0],), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
88
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
89
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
90 self.W = theano.shared(value = initial_W, name = "W")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
91 self.b = theano.shared(value = initial_b, name = "b")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
92
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
93
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
94 initial_b_prime= numpy.zeros((filter_shape[1],))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
95
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
96 self.W_prime=T.dtensor4('W_prime')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
97
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
98 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
99
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
100 self.x = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
101
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
102 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level) * self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
103
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
104 conv1_out = conv.conv2d(self.tilde_x, self.W, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
105 filter_shape=filter_shape, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
106 image_shape=image_shape, border_mode='valid')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
107
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
108
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
109 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
110
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
111
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
112 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
113 filter_shape[3] ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
114 da_image_shape = [ image_shape[0],filter_shape[0],image_shape[2]-filter_shape[2]+1, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
115 image_shape[3]-filter_shape[3]+1 ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
116 initial_W_prime = numpy.asarray( numpy.random.uniform( \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
117 low = -numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
118 high = numpy.sqrt(6./(fan_in+fan_out)), \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
119 size = da_filter_shape), dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
120 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
121
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
122 #import pdb;pdb.set_trace()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
123
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
124 conv2_out = conv.conv2d(self.y, self.W_prime, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
125 filter_shape = da_filter_shape, image_shape = da_image_shape ,\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
126 border_mode='full')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
127
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
128 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
129
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
130 scaled_x = (self.x + center) / scale
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
131
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
132 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
133
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
134 self.cost = T.mean(self.L)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
135
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
136 self.params = [ self.W, self.b, self.b_prime ]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
137
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
138
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
139
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
140 class LeNetConvPoolLayer(object):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
141 def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2,2)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
142 assert image_shape[1]==filter_shape[1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
143 self.input = input
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
144
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
145 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
146 self.W = theano.shared(value = W_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
147
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
148 b_values = numpy.zeros((filter_shape[0],), dtype= theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
149 self.b = theano.shared(value= b_values)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
150
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
151 conv_out = conv.conv2d(input, self.W,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
152 filter_shape=filter_shape, image_shape=image_shape)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
153
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
154
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
155 fan_in = numpy.prod(filter_shape[1:])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
156 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
157
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
158 W_bound = numpy.sqrt(6./(fan_in + fan_out))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
159 self.W.value = numpy.asarray(
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
160 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
161 dtype = theano.config.floatX)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
162
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
163
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
164 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
165
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
166 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
167 self.params = [self.W, self.b]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
168
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
169
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
170 class SdA():
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
171 def __init__(self, input, n_ins_conv, n_ins_mlp, train_set_x, train_set_y, batch_size, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
172 conv_hidden_layers_sizes, mlp_hidden_layers_sizes, corruption_levels, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
173 rng, n_out, pretrain_lr, finetune_lr):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
174
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
175 self.layers = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
176 self.pretrain_functions = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
177 self.params = []
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
178 self.conv_n_layers = len(conv_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
179 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
180
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
181 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
182 self.x = T.dmatrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
183 self.y = T.ivector('y') # the labels are presented as 1D vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
184
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
185
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
186
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
187 for i in xrange( self.conv_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
188
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
189 filter_shape=conv_hidden_layers_sizes[i][0]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
190 image_shape=conv_hidden_layers_sizes[i][1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
191 max_poolsize=conv_hidden_layers_sizes[i][2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
192
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
193 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
194 layer_input=self.x.reshape((batch_size,1,28,28))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
195 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
196 layer_input=self.layers[-1].output
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
197
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
198 layer = LeNetConvPoolLayer(rng, input=layer_input, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
199 image_shape=image_shape, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
200 filter_shape=filter_shape,poolsize=max_poolsize)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
201 print 'Convolutional layer '+str(i+1)+' created'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
202
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
203 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
204 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
205
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
206 da_layer = dA_conv(corruption_level = corruption_levels[0],\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
207 input = layer_input, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
208 shared_W = layer.W, shared_b = layer.b,\
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
209 filter_shape = filter_shape , image_shape = image_shape )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
210
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
211
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
212 gparams = T.grad(da_layer.cost, da_layer.params)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
213
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
214 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
215 for param, gparam in zip(da_layer.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
216 updates[param] = param - gparam * pretrain_lr
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
217
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
218
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
219 update_fn = theano.function([index], da_layer.cost, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
220 updates = updates,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
221 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
222 self.x : train_set_x[index*batch_size:(index+1)*batch_size]} )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
223
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
224 self.pretrain_functions += [update_fn]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
225
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
226 for i in xrange( self.mlp_n_layers ):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
227 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
228 input_size = n_ins_mlp
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
229 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
230 input_size = mlp_hidden_layers_sizes[i-1]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
231
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
232 if i == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
233 if len( self.layers ) == 0 :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
234 layer_input=self.x
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
235 else :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
236 layer_input = self.layers[-1].output.flatten(2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
237 else:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
238 layer_input = self.layers[-1].output
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
239
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
240 layer = SigmoidalLayer(rng, layer_input, input_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
241 mlp_hidden_layers_sizes[i] )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
242
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
243 self.layers += [layer]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
244 self.params += layer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
245
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
246
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
247 print 'MLP layer '+str(i+1)+' created'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
248
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
249 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
250 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
251 self.params += self.logLayer.params
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
252
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
253 cost = self.logLayer.negative_log_likelihood(self.y)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
254
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
255 gparams = T.grad(cost, self.params)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
256 updates = {}
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
257
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
258 for param,gparam in zip(self.params, gparams):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
259 updates[param] = param - gparam*finetune_lr
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
260
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
261 self.finetune = theano.function([index], cost,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
262 updates = updates,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
263 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
264 self.x : train_set_x[index*batch_size:(index+1)*batch_size],
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
265 self.y : train_set_y[index*batch_size:(index+1)*batch_size]} )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
266
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
267
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
268 self.errors = self.logLayer.errors(self.y)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
269
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
270
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
271
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
272 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 2, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
273 pretrain_lr = 0.01, training_epochs = 1000, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
274 dataset='mnist.pkl.gz'):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
275
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
276 f = gzip.open(dataset,'rb')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
277 train_set, valid_set, test_set = cPickle.load(f)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
278 f.close()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
279
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
280
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
281 def shared_dataset(data_xy):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
282 data_x, data_y = data_xy
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
283 shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
284 shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
285 return shared_x, T.cast(shared_y, 'int32')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
286
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
287
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
288 test_set_x, test_set_y = shared_dataset(test_set)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
289 valid_set_x, valid_set_y = shared_dataset(valid_set)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
290 train_set_x, train_set_y = shared_dataset(train_set)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
291
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
292 batch_size = 500 # size of the minibatch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
293
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
294
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
295 n_train_batches = train_set_x.value.shape[0] / batch_size
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
296 n_valid_batches = valid_set_x.value.shape[0] / batch_size
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
297 n_test_batches = test_set_x.value.shape[0] / batch_size
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
298
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
299 # allocate symbolic variables for the data
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
300 index = T.lscalar() # index to a [mini]batch
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
301 x = T.matrix('x') # the data is presented as rasterized images
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
302 y = T.ivector('y') # the labels are presented as 1d vector of
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
303 # [int] labels
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
304 layer0_input = x.reshape((batch_size,1,28,28))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
305
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
306
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
307 # Setup the convolutional layers with their DAs(add as many as you want)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
308 corruption_levels = [ 0.2, 0.2, 0.2]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
309 rng = numpy.random.RandomState(1234)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
310 ker1=2
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
311 ker2=2
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
312 conv_layers=[]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
313 conv_layers.append([[ker1,1,5,5], [batch_size,1,28,28], [2,2] ])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
314 conv_layers.append([[ker2,ker1,5,5], [batch_size,ker1,12,12], [2,2] ])
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
315
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
316 # Setup the MLP layers of the network
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
317 mlp_layers=[500]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
318
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
319 network = SdA(input = layer0_input, n_ins_conv = 28*28, n_ins_mlp = ker2*4*4, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
320 train_set_x = train_set_x, train_set_y = train_set_y, batch_size = batch_size,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
321 conv_hidden_layers_sizes = conv_layers, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
322 mlp_hidden_layers_sizes = mlp_layers, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
323 corruption_levels = corruption_levels , n_out = 10, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
324 rng = rng , pretrain_lr = pretrain_lr , finetune_lr = learning_rate )
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
325
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
326 test_model = theano.function([index], network.errors,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
327 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
328 network.x: test_set_x[index*batch_size:(index+1)*batch_size],
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
329 network.y: test_set_y[index*batch_size:(index+1)*batch_size]})
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
330
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
331 validate_model = theano.function([index], network.errors,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
332 givens = {
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
333 network.x: valid_set_x[index*batch_size:(index+1)*batch_size],
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
334 network.y: valid_set_y[index*batch_size:(index+1)*batch_size]})
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
335
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
336
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
337
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
338 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
339 for i in xrange(len(network.layers)-len(mlp_layers)):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
340 for epoch in xrange(pretraining_epochs):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
341 for batch_index in xrange(n_train_batches):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
342 c = network.pretrain_functions[i](batch_index)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
343 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch),c
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
344
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
345 patience = 10000 # look as this many examples regardless
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
346 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
347 # FOUND
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
348 improvement_threshold = 0.995 # a relative improvement of this much is
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
349
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
350 validation_frequency = min(n_train_batches, patience/2)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
351
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
352
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
353 best_params = None
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
354 best_validation_loss = float('inf')
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
355 test_score = 0.
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
356 start_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
357
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
358 done_looping = False
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
359 epoch = 0
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
360
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
361 while (epoch < training_epochs) and (not done_looping):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
362 epoch = epoch + 1
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
363 for minibatch_index in xrange(n_train_batches):
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
364
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
365 cost_ij = network.finetune(minibatch_index)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
366 iter = epoch * n_train_batches + minibatch_index
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
367
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
368 if (iter+1) % validation_frequency == 0:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
369
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
370 validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
371 this_validation_loss = numpy.mean(validation_losses)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
372 print('epoch %i, minibatch %i/%i, validation error %f %%' % \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
373 (epoch, minibatch_index+1, n_train_batches, \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
374 this_validation_loss*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
375
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
376
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
377 # if we got the best validation score until now
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
378 if this_validation_loss < best_validation_loss:
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
379
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
380 #improve patience if loss improvement is good enough
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
381 if this_validation_loss < best_validation_loss * \
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
382 improvement_threshold :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
383 patience = max(patience, iter * patience_increase)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
384
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
385 # save best validation score and iteration number
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
386 best_validation_loss = this_validation_loss
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
387 best_iter = iter
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
388
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
389 # test it on the test set
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
390 test_losses = [test_model(i) for i in xrange(n_test_batches)]
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
391 test_score = numpy.mean(test_losses)
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
392 print((' epoch %i, minibatch %i/%i, test error of best '
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
393 'model %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
394 (epoch, minibatch_index+1, n_train_batches,
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
395 test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
396
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
397
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
398 if patience <= iter :
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
399 done_looping = True
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
400 break
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
401
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
402 end_time = time.clock()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
403 print(('Optimization complete with best validation score of %f %%,'
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
404 'with test performance %f %%') %
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
405 (best_validation_loss * 100., test_score*100.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
406 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
407
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
408
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
409
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
410
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
411
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
412
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
413 if __name__ == '__main__':
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
414 sgd_optimization_mnist()
128507ac4edf Initial commit for the stacked convolutional denoising autoencoders
Owner <salahmeister@gmail.com>
parents:
diff changeset
415