Mercurial > ift6266

comparison deep/stacked_dae/v2/stacked_dae.py @ 227:acae439d6572

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Ajouté une modification sur stacked_dae qui utilise les nouvelles SeriesTables. Je le met dans le repository pour que mes expériences en cours continuent sans perturbation, et pour que Sylvain puisse récupérer la version actuelle; je fusionnerai à moment donné.
author fsavard
date Fri, 12 Mar 2010 10:31:10 -0500
parents
children 851e7ad4a143
comparison
equal deleted inserted replaced
226:bfe20d63f88c 227:acae439d6572
1 #!/usr/bin/python
2 # coding: utf-8
3
4 import numpy
5 import theano
6 import time
7 import theano.tensor as T
8 from theano.tensor.shared_randomstreams import RandomStreams
9 import copy
10
11 from utils import update_locals
12
13 # taken from LeDeepNet/daa.py
14 # has a special case when taking log(0) (defined =0)
15 # modified to not take the mean anymore
16 from theano.tensor.xlogx import xlogx, xlogy0
17 # it's target*log(output)
18 def binary_cross_entropy(target, output, sum_axis=1):
19 XE = xlogy0(target, output) + xlogy0((1 - target), (1 - output))
20 return -T.sum(XE, axis=sum_axis)
21
22 class LogisticRegression(object):
23 def __init__(self, input, n_in, n_out):
24 # initialize with 0 the weights W as a matrix of shape (n_in, n_out)
25 self.W = theano.shared( value=numpy.zeros((n_in,n_out),
26 dtype = theano.config.floatX) )
27 # initialize the baises b as a vector of n_out 0s
28 self.b = theano.shared( value=numpy.zeros((n_out,),
29 dtype = theano.config.floatX) )
30 # compute vector of class-membership probabilities in symbolic form
31 self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W)+self.b)
32
33 # compute prediction as class whose probability is maximal in
34 # symbolic form
35 self.y_pred=T.argmax(self.p_y_given_x, axis=1)
36
37 # list of parameters for this layer
38 self.params = [self.W, self.b]
39
40 def negative_log_likelihood(self, y):
41 return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y])
42
43 def errors(self, y):
44 # check if y has same dimension of y_pred
45 if y.ndim != self.y_pred.ndim:
46 raise TypeError('y should have the same shape as self.y_pred',
47 ('y', target.type, 'y_pred', self.y_pred.type))
48
49 # check if y is of the correct datatype
50 if y.dtype.startswith('int'):
51 # the T.neq operator returns a vector of 0s and 1s, where 1
52 # represents a mistake in prediction
53 return T.mean(T.neq(self.y_pred, y))
54 else:
55 raise NotImplementedError()
56
57
58 class SigmoidalLayer(object):
59 def __init__(self, rng, input, n_in, n_out):
60 self.input = input
61
62 W_values = numpy.asarray( rng.uniform( \
63 low = -numpy.sqrt(6./(n_in+n_out)), \
64 high = numpy.sqrt(6./(n_in+n_out)), \
65 size = (n_in, n_out)), dtype = theano.config.floatX)
66 self.W = theano.shared(value = W_values)
67
68 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
69 self.b = theano.shared(value= b_values)
70
71 self.output = T.nnet.sigmoid(T.dot(input, self.W) + self.b)
72 self.params = [self.W, self.b]
73
74
75
76 class dA(object):
77 def __init__(self, n_visible= 784, n_hidden= 500, corruption_level = 0.1,\
78 input = None, shared_W = None, shared_b = None):
79 self.n_visible = n_visible
80 self.n_hidden = n_hidden
81
82 # create a Theano random generator that gives symbolic random values
83 theano_rng = RandomStreams()
84
85 if shared_W != None and shared_b != None :
86 self.W = shared_W
87 self.b = shared_b
88 else:
89 # initial values for weights and biases
90 # note : W' was written as `W_prime` and b' as `b_prime`
91
92 # W is initialized with `initial_W` which is uniformely sampled
93 # from -6./sqrt(n_visible+n_hidden) and 6./sqrt(n_hidden+n_visible)
94 # the output of uniform if converted using asarray to dtype
95 # theano.config.floatX so that the code is runable on GPU
96 initial_W = numpy.asarray( numpy.random.uniform( \
97 low = -numpy.sqrt(6./(n_hidden+n_visible)), \
98 high = numpy.sqrt(6./(n_hidden+n_visible)), \
99 size = (n_visible, n_hidden)), dtype = theano.config.floatX)
100 initial_b = numpy.zeros(n_hidden, dtype = theano.config.floatX)
101
102
103 # theano shared variables for weights and biases
104 self.W = theano.shared(value = initial_W, name = "W")
105 self.b = theano.shared(value = initial_b, name = "b")
106
107
108 initial_b_prime= numpy.zeros(n_visible)
109 # tied weights, therefore W_prime is W transpose
110 self.W_prime = self.W.T
111 self.b_prime = theano.shared(value = initial_b_prime, name = "b'")
112
113 # if no input is given, generate a variable representing the input
114 if input == None :
115 # we use a matrix because we expect a minibatch of several examples,
116 # each example being a row
117 self.x = T.dmatrix(name = 'input')
118 else:
119 self.x = input
120 # Equation (1)
121 # keep 90% of the inputs the same and zero-out randomly selected subset of 10% of the inputs
122 # note : first argument of theano.rng.binomial is the shape(size) of
123 # random numbers that it should produce
124 # second argument is the number of trials
125 # third argument is the probability of success of any trial
126 #
127 # this will produce an array of 0s and 1s where 1 has a
128 # probability of 1 - ``corruption_level`` and 0 with
129 # ``corruption_level``
130 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level, dtype=theano.config.floatX) * self.x
131 # Equation (2)
132 # note : y is stored as an attribute of the class so that it can be
133 # used later when stacking dAs.
134 self.y = T.nnet.sigmoid(T.dot(self.tilde_x, self.W ) + self.b)
135 # Equation (3)
136 self.z = T.nnet.sigmoid(T.dot(self.y, self.W_prime) + self.b_prime)
137 # Equation (4)
138 # note : we sum over the size of a datapoint; if we are using minibatches,
139 # L will be a vector, with one entry per example in minibatch
140 #self.L = - T.sum( self.x*T.log(self.z) + (1-self.x)*T.log(1-self.z), axis=1 )
141 #self.L = binary_cross_entropy(target=self.x, output=self.z, sum_axis=1)
142
143 # bypassing z to avoid running to log(0)
144 z_a = T.dot(self.y, self.W_prime) + self.b_prime
145 log_sigmoid = T.log(1) - T.log(1+T.exp(-z_a))
146 # log(1-sigmoid(z_a))
147 log_1_sigmoid = -self.x - T.log(1+T.exp(-z_a))
148 self.L = -T.sum( self.x * (log_sigmoid) \
149 + (1.0-self.x) * (log_1_sigmoid), axis=1 )
150
151 # I added this epsilon to avoid getting log(0) and 1/0 in grad
152 # This means conceptually that there'd be no probability of 0, but that
153 # doesn't seem to me as important (maybe I'm wrong?).
154 #eps = 0.00000001
155 #eps_1 = 1-eps
156 #self.L = - T.sum( self.x * T.log(eps + eps_1*self.z) \
157 # + (1-self.x)*T.log(eps + eps_1*(1-self.z)), axis=1 )
158 # note : L is now a vector, where each element is the cross-entropy cost
159 # of the reconstruction of the corresponding example of the
160 # minibatch. We need to compute the average of all these to get
161 # the cost of the minibatch
162 self.cost = T.mean(self.L)
163
164 self.params = [ self.W, self.b, self.b_prime ]
165
166
167 class SdA(object):
168 def __init__(self, train_set_x, train_set_y, batch_size, n_ins,
169 hidden_layers_sizes, n_outs,
170 corruption_levels, rng, pretrain_lr, finetune_lr, input_divider=1.0):
171 # Just to make sure those are not modified somewhere else afterwards
172 hidden_layers_sizes = copy.deepcopy(hidden_layers_sizes)
173 corruption_levels = copy.deepcopy(corruption_levels)
174
175 update_locals(self, locals())
176
177 self.layers = []
178 self.pretrain_functions = []
179 self.params = []
180 # MODIF: added this so we also get the b_primes
181 # (not used for finetuning... still using ".params")
182 self.all_params = []
183 self.n_layers = len(hidden_layers_sizes)
184
185 print "Creating SdA with params:"
186 print "batch_size", batch_size
187 print "hidden_layers_sizes", hidden_layers_sizes
188 print "corruption_levels", corruption_levels
189 print "n_ins", n_ins
190 print "n_outs", n_outs
191 print "pretrain_lr", pretrain_lr
192 print "finetune_lr", finetune_lr
193 print "input_divider", input_divider
194 print "----"
195
196 self.shared_divider = theano.shared(numpy.asarray(input_divider, dtype=theano.config.floatX))
197
198 if len(hidden_layers_sizes) < 1 :
199 raiseException (' You must have at least one hidden layer ')
200
201
202 # allocate symbolic variables for the data
203 index = T.lscalar() # index to a [mini]batch
204 self.x = T.matrix('x') # the data is presented as rasterized images
205 self.y = T.ivector('y') # the labels are presented as 1D vector of
206 # [int] labels
207
208 for i in xrange( self.n_layers ):
209 # construct the sigmoidal layer
210
211 # the size of the input is either the number of hidden units of
212 # the layer below or the input size if we are on the first layer
213 if i == 0 :
214 input_size = n_ins
215 else:
216 input_size = hidden_layers_sizes[i-1]
217
218 # the input to this layer is either the activation of the hidden
219 # layer below or the input of the SdA if you are on the first
220 # layer
221 if i == 0 :
222 layer_input = self.x
223 else:
224 layer_input = self.layers[-1].output
225
226 layer = SigmoidalLayer(rng, layer_input, input_size,
227 hidden_layers_sizes[i] )
228 # add the layer to the
229 self.layers += [layer]
230 self.params += layer.params
231
232 # Construct a denoising autoencoder that shared weights with this
233 # layer
234 dA_layer = dA(input_size, hidden_layers_sizes[i], \
235 corruption_level = corruption_levels[0],\
236 input = layer_input, \
237 shared_W = layer.W, shared_b = layer.b)
238
239 self.all_params += dA_layer.params
240
241 # Construct a function that trains this dA
242 # compute gradients of layer parameters
243 gparams = T.grad(dA_layer.cost, dA_layer.params)
244 # compute the list of updates
245 updates = {}
246 for param, gparam in zip(dA_layer.params, gparams):
247 updates[param] = param - gparam * pretrain_lr
248
249 # create a function that trains the dA
250 update_fn = theano.function([index], dA_layer.cost, \
251 updates = updates,
252 givens = {
253 self.x : train_set_x[index*batch_size:(index+1)*batch_size] / self.shared_divider})
254 # collect this function into a list
255 self.pretrain_functions += [update_fn]
256
257
258 # We now need to add a logistic layer on top of the MLP
259 self.logLayer = LogisticRegression(\
260 input = self.layers[-1].output,\
261 n_in = hidden_layers_sizes[-1], n_out = n_outs)
262
263 self.params += self.logLayer.params
264 self.all_params += self.logLayer.params
265 # construct a function that implements one step of finetunining
266
267 # compute the cost, defined as the negative log likelihood
268 cost = self.logLayer.negative_log_likelihood(self.y)
269 # compute the gradients with respect to the model parameters
270 gparams = T.grad(cost, self.params)
271 # compute list of updates
272 updates = {}
273 for param,gparam in zip(self.params, gparams):
274 updates[param] = param - gparam*finetune_lr
275
276 self.finetune = theano.function([index], cost,
277 updates = updates,
278 givens = {
279 self.x : train_set_x[index*batch_size:(index+1)*batch_size]/self.shared_divider,
280 self.y : train_set_y[index*batch_size:(index+1)*batch_size]} )
281
282 # symbolic variable that points to the number of errors made on the
283 # minibatch given by self.x and self.y
284
285 self.errors = self.logLayer.errors(self.y)
286
287 if __name__ == '__main__':
288 import sys
289 args = sys.argv[1:]
290