|
371
|
1 #!/usr/bin/python
|
|
|
2 # coding: utf-8
|
|
|
3
|
|
|
4 import numpy
|
|
|
5 import theano
|
|
|
6 import time
|
|
|
7 import theano.tensor as T
|
|
|
8 from theano.tensor.shared_randomstreams import RandomStreams
|
|
|
9 import copy
|
|
|
10
|
|
|
11 from utils import update_locals
|
|
|
12
|
|
|
13 # taken from LeDeepNet/daa.py
|
|
|
14 # has a special case when taking log(0) (defined =0)
|
|
|
15 # modified to not take the mean anymore
|
|
|
16 from theano.tensor.xlogx import xlogx, xlogy0
|
|
|
17 # it's target*log(output)
|
|
|
18 def binary_cross_entropy(target, output, sum_axis=1):
|
|
|
19 XE = xlogy0(target, output) + xlogy0((1 - target), (1 - output))
|
|
|
20 return -T.sum(XE, axis=sum_axis)
|
|
|
21
|
|
|
22 class LogisticRegression(object):
|
|
|
23 def __init__(self, input, n_in, n_out, detection_mode):
|
|
|
24 # initialize with 0 the weights W as a matrix of shape (n_in, n_out)
|
|
|
25 self.W = theano.shared( value=numpy.zeros((n_in,n_out),
|
|
|
26 dtype = theano.config.floatX) )
|
|
|
27 # initialize the baises b as a vector of n_out 0s
|
|
|
28 self.b = theano.shared( value=numpy.zeros((n_out,),
|
|
|
29 dtype = theano.config.floatX) )
|
|
|
30 # compute vector of class-membership probabilities in symbolic form
|
|
|
31 if detection_mode:
|
|
|
32 self.p_y_given_x = T.nnet.sigmoid(T.dot(input, self.W)+self.b)
|
|
|
33 else:
|
|
|
34 self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W)+self.b)
|
|
|
35
|
|
|
36 # compute prediction as class whose probability is maximal in
|
|
|
37 # symbolic form
|
|
|
38 self.y_pred=T.argmax(self.p_y_given_x, axis=1)
|
|
|
39
|
|
|
40 # list of parameters for this layer
|
|
|
41 self.params = [self.W, self.b]
|
|
|
42
|
|
|
43
|
|
|
44 def negative_log_likelihood(self, y):
|
|
|
45 return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y])
|
|
|
46
|
|
|
47 def cross_entropy(self, y):
|
|
|
48 return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y]+T.sum(T.log(1-self.p_y_given_x), axis=1)-T.log(1-self.p_y_given_x)[T.arange(y.shape[0]),y])
|
|
|
49
|
|
|
50 def errors(self, y):
|
|
|
51 # check if y has same dimension of y_pred
|
|
|
52 if y.ndim != self.y_pred.ndim:
|
|
|
53 raise TypeError('y should have the same shape as self.y_pred',
|
|
|
54 ('y', target.type, 'y_pred', self.y_pred.type))
|
|
|
55
|
|
|
56 # check if y is of the correct datatype
|
|
|
57 if y.dtype.startswith('int'):
|
|
|
58 # the T.neq operator returns a vector of 0s and 1s, where 1
|
|
|
59 # represents a mistake in prediction
|
|
|
60 return T.mean(T.neq(self.y_pred, y))
|
|
|
61 else:
|
|
|
62 raise NotImplementedError()
|
|
|
63
|
|
|
64
|
|
|
65 class SigmoidalLayer(object):
|
|
|
66 def __init__(self, rng, input, n_in, n_out):
|
|
|
67 self.input = input
|
|
|
68
|
|
|
69 W_values = numpy.asarray( rng.uniform( \
|
|
|
70 low = -numpy.sqrt(6./(n_in+n_out)), \
|
|
|
71 high = numpy.sqrt(6./(n_in+n_out)), \
|
|
|
72 size = (n_in, n_out)), dtype = theano.config.floatX)
|
|
|
73 self.W = theano.shared(value = W_values)
|
|
|
74
|
|
|
75 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
|
|
|
76 self.b = theano.shared(value= b_values)
|
|
|
77
|
|
|
78 self.output = T.nnet.sigmoid(T.dot(input, self.W) + self.b)
|
|
|
79 self.params = [self.W, self.b]
|
|
|
80
|
|
|
81
|
|
|
82
|
|
|
83 class dA(object):
|
|
|
84 def __init__(self, n_visible= 784, n_hidden= 500, corruption_level = 0.1,\
|
|
|
85 input = None, shared_W = None, shared_b = None):
|
|
|
86 self.n_visible = n_visible
|
|
|
87 self.n_hidden = n_hidden
|
|
|
88
|
|
|
89 # create a Theano random generator that gives symbolic random values
|
|
|
90 theano_rng = RandomStreams()
|
|
|
91
|
|
|
92 if shared_W != None and shared_b != None :
|
|
|
93 self.W = shared_W
|
|
|
94 self.b = shared_b
|
|
|
95 else:
|
|
|
96 # initial values for weights and biases
|
|
|
97 # note : W' was written as `W_prime` and b' as `b_prime`
|
|
|
98
|
|
|
99 # W is initialized with `initial_W` which is uniformely sampled
|
|
|
100 # from -6./sqrt(n_visible+n_hidden) and 6./sqrt(n_hidden+n_visible)
|
|
|
101 # the output of uniform if converted using asarray to dtype
|
|
|
102 # theano.config.floatX so that the code is runable on GPU
|
|
|
103 initial_W = numpy.asarray( numpy.random.uniform( \
|
|
|
104 low = -numpy.sqrt(6./(n_hidden+n_visible)), \
|
|
|
105 high = numpy.sqrt(6./(n_hidden+n_visible)), \
|
|
|
106 size = (n_visible, n_hidden)), dtype = theano.config.floatX)
|
|
|
107 initial_b = numpy.zeros(n_hidden, dtype = theano.config.floatX)
|
|
|
108
|
|
|
109
|
|
|
110 # theano shared variables for weights and biases
|
|
|
111 self.W = theano.shared(value = initial_W, name = "W")
|
|
|
112 self.b = theano.shared(value = initial_b, name = "b")
|
|
|
113
|
|
|
114
|
|
|
115 initial_b_prime= numpy.zeros(n_visible)
|
|
|
116 # tied weights, therefore W_prime is W transpose
|
|
|
117 self.W_prime = self.W.T
|
|
|
118 self.b_prime = theano.shared(value = initial_b_prime, name = "b'")
|
|
|
119
|
|
|
120 # if no input is given, generate a variable representing the input
|
|
|
121 if input == None :
|
|
|
122 # we use a matrix because we expect a minibatch of several examples,
|
|
|
123 # each example being a row
|
|
|
124 self.x = T.matrix(name = 'input')
|
|
|
125 else:
|
|
|
126 self.x = input
|
|
|
127 # Equation (1)
|
|
|
128 # keep 90% of the inputs the same and zero-out randomly selected subset of 10% of the inputs
|
|
|
129 # note : first argument of theano.rng.binomial is the shape(size) of
|
|
|
130 # random numbers that it should produce
|
|
|
131 # second argument is the number of trials
|
|
|
132 # third argument is the probability of success of any trial
|
|
|
133 #
|
|
|
134 # this will produce an array of 0s and 1s where 1 has a
|
|
|
135 # probability of 1 - ``corruption_level`` and 0 with
|
|
|
136 # ``corruption_level``
|
|
|
137 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level, dtype=theano.config.floatX) * self.x
|
|
|
138 # Equation (2)
|
|
|
139 # note : y is stored as an attribute of the class so that it can be
|
|
|
140 # used later when stacking dAs.
|
|
|
141 self.y = T.nnet.sigmoid(T.dot(self.tilde_x, self.W ) + self.b)
|
|
|
142 # Equation (3)
|
|
|
143 #self.z = T.nnet.sigmoid(T.dot(self.y, self.W_prime) + self.b_prime)
|
|
|
144 # Equation (4)
|
|
|
145 # note : we sum over the size of a datapoint; if we are using minibatches,
|
|
|
146 # L will be a vector, with one entry per example in minibatch
|
|
|
147 #self.L = - T.sum( self.x*T.log(self.z) + (1-self.x)*T.log(1-self.z), axis=1 )
|
|
|
148 #self.L = binary_cross_entropy(target=self.x, output=self.z, sum_axis=1)
|
|
|
149
|
|
|
150 # bypassing z to avoid running to log(0)
|
|
|
151 z_a = T.dot(self.y, self.W_prime) + self.b_prime
|
|
|
152 log_sigmoid = T.log(1.) - T.log(1.+T.exp(-z_a))
|
|
|
153 # log(1-sigmoid(z_a))
|
|
|
154 log_1_sigmoid = -z_a - T.log(1.+T.exp(-z_a))
|
|
|
155 self.L = -T.sum( self.x * (log_sigmoid) \
|
|
|
156 + (1.0-self.x) * (log_1_sigmoid), axis=1 )
|
|
|
157
|
|
|
158 # I added this epsilon to avoid getting log(0) and 1/0 in grad
|
|
|
159 # This means conceptually that there'd be no probability of 0, but that
|
|
|
160 # doesn't seem to me as important (maybe I'm wrong?).
|
|
|
161 #eps = 0.00000001
|
|
|
162 #eps_1 = 1-eps
|
|
|
163 #self.L = - T.sum( self.x * T.log(eps + eps_1*self.z) \
|
|
|
164 # + (1-self.x)*T.log(eps + eps_1*(1-self.z)), axis=1 )
|
|
|
165 # note : L is now a vector, where each element is the cross-entropy cost
|
|
|
166 # of the reconstruction of the corresponding example of the
|
|
|
167 # minibatch. We need to compute the average of all these to get
|
|
|
168 # the cost of the minibatch
|
|
|
169 self.cost = T.mean(self.L)
|
|
|
170
|
|
|
171 self.params = [ self.W, self.b, self.b_prime ]
|
|
|
172
|
|
|
173
|
|
|
174 class SdA(object):
|
|
|
175 def __init__(self, batch_size, n_ins,
|
|
|
176 hidden_layers_sizes, n_outs,
|
|
|
177 corruption_levels, rng, pretrain_lr, finetune_lr, detection_mode):
|
|
|
178 # Just to make sure those are not modified somewhere else afterwards
|
|
|
179 hidden_layers_sizes = copy.deepcopy(hidden_layers_sizes)
|
|
|
180 corruption_levels = copy.deepcopy(corruption_levels)
|
|
|
181
|
|
|
182 update_locals(self, locals())
|
|
|
183
|
|
|
184 self.layers = []
|
|
|
185 self.pretrain_functions = []
|
|
|
186 self.params = []
|
|
|
187 # MODIF: added this so we also get the b_primes
|
|
|
188 # (not used for finetuning... still using ".params")
|
|
|
189 self.all_params = []
|
|
|
190 self.n_layers = len(hidden_layers_sizes)
|
|
|
191 self.logistic_params = []
|
|
|
192
|
|
|
193 print "Creating SdA with params:"
|
|
|
194 print "batch_size", batch_size
|
|
|
195 print "hidden_layers_sizes", hidden_layers_sizes
|
|
|
196 print "corruption_levels", corruption_levels
|
|
|
197 print "n_ins", n_ins
|
|
|
198 print "n_outs", n_outs
|
|
|
199 print "pretrain_lr", pretrain_lr
|
|
|
200 print "finetune_lr", finetune_lr
|
|
|
201 print "detection_mode", detection_mode
|
|
|
202 print "----"
|
|
|
203
|
|
|
204 if len(hidden_layers_sizes) < 1 :
|
|
|
205 raiseException (' You must have at least one hidden layer ')
|
|
|
206
|
|
|
207
|
|
|
208 # allocate symbolic variables for the data
|
|
|
209 #index = T.lscalar() # index to a [mini]batch
|
|
|
210 self.x = T.matrix('x') # the data is presented as rasterized images
|
|
|
211 self.y = T.ivector('y') # the labels are presented as 1D vector of
|
|
|
212 # [int] labels
|
|
|
213 self.finetune_lr = T.fscalar('finetune_lr') #To get a dynamic finetune learning rate
|
|
|
214
|
|
|
215 for i in xrange( self.n_layers ):
|
|
|
216 # construct the sigmoidal layer
|
|
|
217
|
|
|
218 # the size of the input is either the number of hidden units of
|
|
|
219 # the layer below or the input size if we are on the first layer
|
|
|
220 if i == 0 :
|
|
|
221 input_size = n_ins
|
|
|
222 else:
|
|
|
223 input_size = hidden_layers_sizes[i-1]
|
|
|
224
|
|
|
225 # the input to this layer is either the activation of the hidden
|
|
|
226 # layer below or the input of the SdA if you are on the first
|
|
|
227 # layer
|
|
|
228 if i == 0 :
|
|
|
229 layer_input = self.x
|
|
|
230 else:
|
|
|
231 layer_input = self.layers[-1].output
|
|
|
232
|
|
|
233 layer = SigmoidalLayer(rng, layer_input, input_size,
|
|
|
234 hidden_layers_sizes[i] )
|
|
|
235 # add the layer to the
|
|
|
236 self.layers += [layer]
|
|
|
237 self.params += layer.params
|
|
|
238
|
|
|
239 # Construct a denoising autoencoder that shared weights with this
|
|
|
240 # layer
|
|
|
241 dA_layer = dA(input_size, hidden_layers_sizes[i], \
|
|
|
242 corruption_level = corruption_levels[0],\
|
|
|
243 input = layer_input, \
|
|
|
244 shared_W = layer.W, shared_b = layer.b)
|
|
|
245
|
|
|
246 self.all_params += dA_layer.params
|
|
|
247
|
|
|
248 # Construct a function that trains this dA
|
|
|
249 # compute gradients of layer parameters
|
|
|
250 gparams = T.grad(dA_layer.cost, dA_layer.params)
|
|
|
251 # compute the list of updates
|
|
|
252 updates = {}
|
|
|
253 for param, gparam in zip(dA_layer.params, gparams):
|
|
|
254 updates[param] = param - gparam * pretrain_lr
|
|
|
255
|
|
|
256 # create a function that trains the dA
|
|
|
257 update_fn = theano.function([self.x], dA_layer.cost, \
|
|
|
258 updates = updates)#,
|
|
|
259 # givens = {
|
|
|
260 # self.x : ensemble})
|
|
|
261 # collect this function into a list
|
|
|
262 #update_fn = theano.function([index], dA_layer.cost, \
|
|
|
263 # updates = updates,
|
|
|
264 # givens = {
|
|
|
265 # self.x : train_set_x[index*batch_size:(index+1)*batch_size] / self.shared_divider})
|
|
|
266 # collect this function into a list
|
|
|
267 self.pretrain_functions += [update_fn]
|
|
|
268
|
|
|
269
|
|
|
270 # We now need to add a logistic layer on top of the SDA
|
|
|
271 self.logLayer = LogisticRegression(\
|
|
|
272 input = self.layers[-1].output,\
|
|
|
273 n_in = hidden_layers_sizes[-1], n_out = n_outs, detection_mode = detection_mode)
|
|
|
274
|
|
|
275 self.params += self.logLayer.params
|
|
|
276 self.all_params += self.logLayer.params
|
|
|
277 # construct a function that implements one step of finetunining
|
|
|
278
|
|
|
279
|
|
|
280 if detection_mode:
|
|
|
281 # compute the cost, defined as the negative log likelihood
|
|
|
282 cost = self.logLayer.cross_entropy(self.y)
|
|
|
283 # compute the gradients with respect to the logistic regression parameters
|
|
|
284 gparams = T.grad(cost, self.logLayer.params)
|
|
|
285 # compute list of updates
|
|
|
286 updates = {}
|
|
|
287 for param,gparam in zip(self.logLayer.params, gparams):
|
|
|
288 updates[param] = param - gparam*finetune_lr
|
|
|
289
|
|
|
290 else:
|
|
|
291 # compute the cost, defined as the negative log likelihood
|
|
|
292 cost = self.logLayer.negative_log_likelihood(self.y)
|
|
|
293 # compute the gradients with respect to the model parameters
|
|
|
294 gparams = T.grad(cost, self.params)
|
|
|
295 # compute list of updates
|
|
|
296 updates = {}
|
|
|
297 for param,gparam in zip(self.params, gparams):
|
|
|
298 updates[param] = param - gparam*self.finetune_lr
|
|
|
299
|
|
|
300 self.finetune = theano.function([self.x,self.y,self.finetune_lr], cost,
|
|
|
301 updates = updates)#,
|
|
|
302
|
|
|
303 # symbolic variable that points to the number of errors made on the
|
|
|
304 # minibatch given by self.x and self.y
|
|
|
305
|
|
|
306 self.errors = self.logLayer.errors(self.y)
|
|
|
307
|
|
|
308
|
|
|
309 #STRUCTURE FOR THE FINETUNING OF THE LOGISTIC REGRESSION ON THE TOP WITH
|
|
|
310 #ALL HIDDEN LAYERS AS INPUT
|
|
|
311 '''
|
|
|
312
|
|
|
313 all_h=[]
|
|
|
314 for i in xrange(self.n_layers):
|
|
|
315 all_h.append(self.layers[i].output)
|
|
|
316 self.all_hidden=T.concatenate(all_h,axis=1)
|
|
|
317
|
|
|
318
|
|
|
319 self.logLayer2 = LogisticRegression(\
|
|
|
320 input = self.all_hidden,\
|
|
|
321 n_in = sum(hidden_layers_sizes), n_out = n_outs)
|
|
|
322 #n_in=hidden_layers_sizes[0],n_out=n_outs)
|
|
|
323
|
|
|
324 #self.logistic_params+= self.logLayer2.params
|
|
|
325 # construct a function that implements one step of finetunining
|
|
|
326
|
|
|
327 self.logistic_params+=self.logLayer2.params
|
|
|
328 # compute the cost, defined as the negative log likelihood
|
|
|
329 if DETECTION_MODE:
|
|
|
330 cost2 = self.logLayer2.cross_entropy(self.y)
|
|
|
331 else:
|
|
|
332 cost2 = self.logLayer2.negative_log_likelihood(self.y)
|
|
|
333 # compute the gradients with respect to the model parameters
|
|
|
334 gparams2 = T.grad(cost2, self.logistic_params)
|
|
|
335
|
|
|
336 # compute list of updates
|
|
|
337 updates2 = {}
|
|
|
338 for param,gparam in zip(self.logistic_params, gparams2):
|
|
|
339 updates2[param] = param - gparam*finetune_lr
|
|
|
340
|
|
|
341 self.finetune2 = theano.function([self.x,self.y], cost2,
|
|
|
342 updates = updates2)
|
|
|
343
|
|
|
344 # symbolic variable that points to the number of errors made on the
|
|
|
345 # minibatch given by self.x and self.y
|
|
|
346
|
|
|
347 self.errors2 = self.logLayer2.errors(self.y)
|
|
|
348 '''
|
|
|
349
|
|
|
350 if __name__ == '__main__':
|
|
|
351 import sys
|
|
|
352 args = sys.argv[1:]
|
|
|
353
|
