Mercurial > ift6266

comparison deep/convolutional_dae/stacked_convolutional_dae.py @ 200:3f2cc90ad51c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Adapt the sdae code for ift6266.datasets input.
author Arnaud Bergeron <abergeron@gmail.com>
date Tue, 02 Mar 2010 20:16:30 -0500
parents 1f5937e9e530
children 334d2444000d
comparison
equal deleted inserted replaced
199:777f48ba30df 200:3f2cc90ad51c
5 from theano.tensor.shared_randomstreams import RandomStreams 5 from theano.tensor.shared_randomstreams import RandomStreams
6 import theano.sandbox.softsign 6 import theano.sandbox.softsign
7 7
8 from theano.tensor.signal import downsample 8 from theano.tensor.signal import downsample
9 from theano.tensor.nnet import conv 9 from theano.tensor.nnet import conv
10 import gzip 10
11 import cPickle 11 from ift6266 import datasets
12 12
13 13 from ift6266.baseline.log_reg.log_reg import LogisticRegression
14 class LogisticRegression(object):
15
16 def __init__(self, input, n_in, n_out):
17
18 self.W = theano.shared( value=numpy.zeros((n_in,n_out),
19 dtype = theano.config.floatX) )
20
21 self.b = theano.shared( value=numpy.zeros((n_out,),
22 dtype = theano.config.floatX) )
23
24 self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W)+self.b)
25
26
27 self.y_pred=T.argmax(self.p_y_given_x, axis=1)
28
29 self.params = [self.W, self.b]
30
31 def negative_log_likelihood(self, y):
32 return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y])
33
34 def MSE(self, y):
35 return -T.mean(abs((self.p_y_given_x)[T.arange(y.shape[0]),y]-y)**2)
36
37 def errors(self, y):
38 if y.ndim != self.y_pred.ndim:
39 raise TypeError('y should have the same shape as self.y_pred',
40 ('y', target.type, 'y_pred', self.y_pred.type))
41
42
43 if y.dtype.startswith('int'):
44 return T.mean(T.neq(self.y_pred, y))
45 else:
46 raise NotImplementedError()
47
48 14
49 class SigmoidalLayer(object): 15 class SigmoidalLayer(object):
50 def __init__(self, rng, input, n_in, n_out): 16 def __init__(self, rng, input, n_in, n_out):
51 17
52 self.input = input 18 self.input = input
63 self.output = T.tanh(T.dot(input, self.W) + self.b) 29 self.output = T.tanh(T.dot(input, self.W) + self.b)
64 self.params = [self.W, self.b] 30 self.params = [self.W, self.b]
65 31
66 class dA_conv(object): 32 class dA_conv(object):
67 33
68 def __init__(self, corruption_level = 0.1, input = None, shared_W = None,\ 34 def __init__(self, input, filter_shape, corruption_level = 0.1,
69 shared_b = None, filter_shape = None, image_shape = None, poolsize = (2,2)): 35 shared_W = None, shared_b = None, image_shape = None,
36 poolsize = (2,2)):
70 37
71 theano_rng = RandomStreams() 38 theano_rng = RandomStreams()
72 39
73 fan_in = numpy.prod(filter_shape[1:]) 40 fan_in = numpy.prod(filter_shape[1:])
74 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) 41 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
78 45
79 if shared_W != None and shared_b != None : 46 if shared_W != None and shared_b != None :
80 self.W = shared_W 47 self.W = shared_W
81 self.b = shared_b 48 self.b = shared_b
82 else: 49 else:
83 initial_W = numpy.asarray( numpy.random.uniform( \ 50 initial_W = numpy.asarray( numpy.random.uniform(
84 low = -numpy.sqrt(6./(fan_in+fan_out)), \ 51 low = -numpy.sqrt(6./(fan_in+fan_out)),
85 high = numpy.sqrt(6./(fan_in+fan_out)), \ 52 high = numpy.sqrt(6./(fan_in+fan_out)),
86 size = filter_shape), dtype = theano.config.floatX) 53 size = filter_shape), dtype = theano.config.floatX)
87 initial_b = numpy.zeros((filter_shape[0],), dtype= theano.config.floatX) 54 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
88
89
90 self.W = theano.shared(value = initial_W, name = "W") 55 self.W = theano.shared(value = initial_W, name = "W")
91 self.b = theano.shared(value = initial_b, name = "b") 56 self.b = theano.shared(value = initial_b, name = "b")
92 57
93 58
94 initial_b_prime= numpy.zeros((filter_shape[1],)) 59 initial_b_prime= numpy.zeros((filter_shape[1],))
99 64
100 self.x = input 65 self.x = input
101 66
102 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level) * self.x 67 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level) * self.x
103 68
104 conv1_out = conv.conv2d(self.tilde_x, self.W, \ 69 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
105 filter_shape=filter_shape, \ 70 image_shape=image_shape, border_mode='valid')
106 image_shape=image_shape, border_mode='valid')
107 71
108 72
109 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x')) 73 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
110 74
111 75
112 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\ 76 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
113 filter_shape[3] ] 77 filter_shape[3] ]
114 da_image_shape = [ image_shape[0],filter_shape[0],image_shape[2]-filter_shape[2]+1, \
115 image_shape[3]-filter_shape[3]+1 ]
116 initial_W_prime = numpy.asarray( numpy.random.uniform( \ 78 initial_W_prime = numpy.asarray( numpy.random.uniform( \
117 low = -numpy.sqrt(6./(fan_in+fan_out)), \ 79 low = -numpy.sqrt(6./(fan_in+fan_out)), \
118 high = numpy.sqrt(6./(fan_in+fan_out)), \ 80 high = numpy.sqrt(6./(fan_in+fan_out)), \
119 size = da_filter_shape), dtype = theano.config.floatX) 81 size = da_filter_shape), dtype = theano.config.floatX)
120 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime") 82 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
121 83
122 #import pdb;pdb.set_trace() 84 conv2_out = conv.conv2d(self.y, self.W_prime,
123 85 filter_shape = da_filter_shape,
124 conv2_out = conv.conv2d(self.y, self.W_prime, \ 86 border_mode='full')
125 filter_shape = da_filter_shape, image_shape = da_image_shape ,\
126 border_mode='full')
127 87
128 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale 88 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
129 89
130 scaled_x = (self.x + center) / scale 90 scaled_x = (self.x + center) / scale
131 91
132 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 ) 92 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
133 93
134 self.cost = T.mean(self.L) 94 self.cost = T.mean(self.L)
135 95
136 self.params = [ self.W, self.b, self.b_prime ] 96 self.params = [ self.W, self.b, self.b_prime ]
137
138
139 97
140 class LeNetConvPoolLayer(object): 98 class LeNetConvPoolLayer(object):
141 def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2,2)): 99 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
142 assert image_shape[1]==filter_shape[1]
143 self.input = input 100 self.input = input
144 101
145 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX) 102 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
146 self.W = theano.shared(value = W_values) 103 self.W = theano.shared(value=W_values)
147 104
148 b_values = numpy.zeros((filter_shape[0],), dtype= theano.config.floatX) 105 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
149 self.b = theano.shared(value= b_values) 106 self.b = theano.shared(value=b_values)
150 107
151 conv_out = conv.conv2d(input, self.W, 108 conv_out = conv.conv2d(input, self.W,
152 filter_shape=filter_shape, image_shape=image_shape) 109 filter_shape=filter_shape, image_shape=image_shape)
153 110
154 111
166 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) 123 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
167 self.params = [self.W, self.b] 124 self.params = [self.W, self.b]
168 125
169 126
170 class SdA(): 127 class SdA():
171 def __init__(self, input, n_ins_conv, n_ins_mlp, train_set_x, train_set_y, batch_size, \ 128 def __init__(self, input, n_ins_mlp, conv_hidden_layers_sizes,
172 conv_hidden_layers_sizes, mlp_hidden_layers_sizes, corruption_levels, \ 129 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
173 rng, n_out, pretrain_lr, finetune_lr): 130 pretrain_lr, finetune_lr):
174 131
175 self.layers = [] 132 self.layers = []
176 self.pretrain_functions = [] 133 self.pretrain_functions = []
177 self.params = [] 134 self.params = []
178 self.conv_n_layers = len(conv_hidden_layers_sizes) 135 self.conv_n_layers = len(conv_hidden_layers_sizes)
179 self.mlp_n_layers = len(mlp_hidden_layers_sizes) 136 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
180 137
181 index = T.lscalar() # index to a [mini]batch
182 self.x = T.dmatrix('x') # the data is presented as rasterized images 138 self.x = T.dmatrix('x') # the data is presented as rasterized images
183 self.y = T.ivector('y') # the labels are presented as 1D vector of 139 self.y = T.ivector('y') # the labels are presented as 1D vector of
184 140
185
186
187 for i in xrange( self.conv_n_layers ): 141 for i in xrange( self.conv_n_layers ):
188
189 filter_shape=conv_hidden_layers_sizes[i][0] 142 filter_shape=conv_hidden_layers_sizes[i][0]
190 image_shape=conv_hidden_layers_sizes[i][1] 143 image_shape=conv_hidden_layers_sizes[i][1]
191 max_poolsize=conv_hidden_layers_sizes[i][2] 144 max_poolsize=conv_hidden_layers_sizes[i][2]
192 145
193 if i == 0 : 146 if i == 0 :
194 layer_input=self.x.reshape((batch_size,1,28,28)) 147 layer_input=self.x.reshape((self.x.shape[0], 1, 32, 32))
195 else: 148 else:
196 layer_input=self.layers[-1].output 149 layer_input=self.layers[-1].output
197 150
198 layer = LeNetConvPoolLayer(rng, input=layer_input, \ 151 layer = LeNetConvPoolLayer(rng, input=layer_input,
199 image_shape=image_shape, \ 152 image_shape=image_shape,
200 filter_shape=filter_shape,poolsize=max_poolsize) 153 filter_shape=filter_shape,
201 print 'Convolutional layer '+str(i+1)+' created' 154 poolsize=max_poolsize)
202 155 print 'Convolutional layer', str(i+1), 'created'
156
203 self.layers += [layer] 157 self.layers += [layer]
204 self.params += layer.params 158 self.params += layer.params
205 159
206 da_layer = dA_conv(corruption_level = corruption_levels[0],\ 160 da_layer = dA_conv(corruption_level = corruption_levels[0],
207 input = layer_input, \ 161 input = layer_input,
208 shared_W = layer.W, shared_b = layer.b,\ 162 shared_W = layer.W, shared_b = layer.b,
209 filter_shape = filter_shape , image_shape = image_shape ) 163 filter_shape = filter_shape,
210 164 image_shape = image_shape )
211 165
212 gparams = T.grad(da_layer.cost, da_layer.params) 166 gparams = T.grad(da_layer.cost, da_layer.params)
213 167
214 updates = {} 168 updates = {}
215 for param, gparam in zip(da_layer.params, gparams): 169 for param, gparam in zip(da_layer.params, gparams):
216 updates[param] = param - gparam * pretrain_lr 170 updates[param] = param - gparam * pretrain_lr
217 171
218 172 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
219 update_fn = theano.function([index], da_layer.cost, \ 173
220 updates = updates,
221 givens = {
222 self.x : train_set_x[index*batch_size:(index+1)*batch_size]} )
223
224 self.pretrain_functions += [update_fn] 174 self.pretrain_functions += [update_fn]
225 175
226 for i in xrange( self.mlp_n_layers ): 176 for i in xrange( self.mlp_n_layers ):
227 if i == 0 : 177 if i == 0 :
228 input_size = n_ins_mlp 178 input_size = n_ins_mlp
229 else: 179 else:
230 input_size = mlp_hidden_layers_sizes[i-1] 180 input_size = mlp_hidden_layers_sizes[i-1]
231 181
232 if i == 0 : 182 if i == 0 :
233 if len( self.layers ) == 0 : 183 if len( self.layers ) == 0 :
234 layer_input=self.x 184 layer_input=self.x
235 else : 185 else :
236 layer_input = self.layers[-1].output.flatten(2) 186 layer_input = self.layers[-1].output.flatten(2)
237 else: 187 else:
238 layer_input = self.layers[-1].output 188 layer_input = self.layers[-1].output
239 189
240 layer = SigmoidalLayer(rng, layer_input, input_size, 190 layer = SigmoidalLayer(rng, layer_input, input_size,
241 mlp_hidden_layers_sizes[i] ) 191 mlp_hidden_layers_sizes[i] )
242 192
243 self.layers += [layer] 193 self.layers += [layer]
244 self.params += layer.params 194 self.params += layer.params
245 195
246 196 print 'MLP layer', str(i+1), 'created'
247 print 'MLP layer '+str(i+1)+' created'
248 197
249 self.logLayer = LogisticRegression(input=self.layers[-1].output, \ 198 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
250 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out) 199 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
251 self.params += self.logLayer.params 200 self.params += self.logLayer.params
252 201
253 cost = self.logLayer.negative_log_likelihood(self.y) 202 cost = self.logLayer.negative_log_likelihood(self.y)
254 203
255 gparams = T.grad(cost, self.params) 204 gparams = T.grad(cost, self.params)
205
256 updates = {} 206 updates = {}
257
258 for param,gparam in zip(self.params, gparams): 207 for param,gparam in zip(self.params, gparams):
259 updates[param] = param - gparam*finetune_lr 208 updates[param] = param - gparam*finetune_lr
260 209
261 self.finetune = theano.function([index], cost, 210 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
262 updates = updates, 211
263 givens = {
264 self.x : train_set_x[index*batch_size:(index+1)*batch_size],
265 self.y : train_set_y[index*batch_size:(index+1)*batch_size]} )
266
267
268 self.errors = self.logLayer.errors(self.y) 212 self.errors = self.logLayer.errors(self.y)
269 213
270
271
272 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 2, \ 214 def sgd_optimization_mnist( learning_rate=0.1, pretraining_epochs = 2, \
273 pretrain_lr = 0.01, training_epochs = 1000, \ 215 pretrain_lr = 0.01, training_epochs = 1000, \
274 dataset='mnist.pkl.gz'): 216 dataset=datasets.nist_digits):
275 217
276 f = gzip.open(dataset,'rb')
277 train_set, valid_set, test_set = cPickle.load(f)
278 f.close()
279
280
281 def shared_dataset(data_xy):
282 data_x, data_y = data_xy
283 shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX))
284 shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX))
285 return shared_x, T.cast(shared_y, 'int32')
286
287
288 test_set_x, test_set_y = shared_dataset(test_set)
289 valid_set_x, valid_set_y = shared_dataset(valid_set)
290 train_set_x, train_set_y = shared_dataset(train_set)
291
292 batch_size = 500 # size of the minibatch 218 batch_size = 500 # size of the minibatch
293
294
295 n_train_batches = train_set_x.value.shape[0] / batch_size
296 n_valid_batches = valid_set_x.value.shape[0] / batch_size
297 n_test_batches = test_set_x.value.shape[0] / batch_size
298 219
299 # allocate symbolic variables for the data 220 # allocate symbolic variables for the data
300 index = T.lscalar() # index to a [mini]batch 221 index = T.lscalar() # index to a [mini]batch
301 x = T.matrix('x') # the data is presented as rasterized images 222 x = T.matrix('x') # the data is presented as rasterized images
302 y = T.ivector('y') # the labels are presented as 1d vector of 223 y = T.ivector('y') # the labels are presented as 1d vector of
303 # [int] labels 224 # [int] labels
304 layer0_input = x.reshape((batch_size,1,28,28)) 225 layer0_input = x.reshape((x.shape[0],1,32,32))
305 226
306 227
307 # Setup the convolutional layers with their DAs(add as many as you want) 228 # Setup the convolutional layers with their DAs(add as many as you want)
308 corruption_levels = [ 0.2, 0.2, 0.2] 229 corruption_levels = [ 0.2, 0.2, 0.2]
309 rng = numpy.random.RandomState(1234) 230 rng = numpy.random.RandomState(1234)
310 ker1=2 231 ker1=2
311 ker2=2 232 ker2=2
312 conv_layers=[] 233 conv_layers=[]
313 conv_layers.append([[ker1,1,5,5], [batch_size,1,28,28], [2,2] ]) 234 conv_layers.append([[ker1,1,5,5], None, [2,2] ])
314 conv_layers.append([[ker2,ker1,5,5], [batch_size,ker1,12,12], [2,2] ]) 235 conv_layers.append([[ker2,ker1,5,5], None, [2,2] ])
315 236
316 # Setup the MLP layers of the network 237 # Setup the MLP layers of the network
317 mlp_layers=[500] 238 mlp_layers=[500]
318 239
319 network = SdA(input = layer0_input, n_ins_conv = 28*28, n_ins_mlp = ker2*4*4, \ 240 network = SdA(input = layer0_input, n_ins_mlp = ker2*4*4,
320 train_set_x = train_set_x, train_set_y = train_set_y, batch_size = batch_size, 241 conv_hidden_layers_sizes = conv_layers,
321 conv_hidden_layers_sizes = conv_layers, \ 242 mlp_hidden_layers_sizes = mlp_layers,
322 mlp_hidden_layers_sizes = mlp_layers, \ 243 corruption_levels = corruption_levels , n_out = 10,
323 corruption_levels = corruption_levels , n_out = 10, \ 244 rng = rng , pretrain_lr = pretrain_lr ,
324 rng = rng , pretrain_lr = pretrain_lr , finetune_lr = learning_rate ) 245 finetune_lr = learning_rate )
325 246
326 test_model = theano.function([index], network.errors, 247 test_model = theano.function([network.x, network.y], network.errors)
327 givens = { 248
328 network.x: test_set_x[index*batch_size:(index+1)*batch_size],
329 network.y: test_set_y[index*batch_size:(index+1)*batch_size]})
330
331 validate_model = theano.function([index], network.errors,
332 givens = {
333 network.x: valid_set_x[index*batch_size:(index+1)*batch_size],
334 network.y: valid_set_y[index*batch_size:(index+1)*batch_size]})
335
336
337
338 start_time = time.clock() 249 start_time = time.clock()
339 for i in xrange(len(network.layers)-len(mlp_layers)): 250 for i in xrange(len(network.layers)-len(mlp_layers)):
340 for epoch in xrange(pretraining_epochs): 251 for epoch in xrange(pretraining_epochs):
341 for batch_index in xrange(n_train_batches): 252 for x, y in dataset.train(batch_size):
342 c = network.pretrain_functions[i](batch_index) 253 c = network.pretrain_functions[i](x)
343 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch),c 254 print 'pre-training convolution layer %i, epoch %d, cost '%(i,epoch), c
344 255
345 patience = 10000 # look as this many examples regardless 256 patience = 10000 # look as this many examples regardless
346 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS 257 patience_increase = 2. # WAIT THIS MUCH LONGER WHEN A NEW BEST IS
347 # FOUND 258 # FOUND
348 improvement_threshold = 0.995 # a relative improvement of this much is 259 improvement_threshold = 0.995 # a relative improvement of this much is
349 260
350 validation_frequency = min(n_train_batches, patience/2) 261 validation_frequency = patience/2
351
352 262
353 best_params = None 263 best_params = None
354 best_validation_loss = float('inf') 264 best_validation_loss = float('inf')
355 test_score = 0. 265 test_score = 0.
356 start_time = time.clock() 266 start_time = time.clock()
357 267
358 done_looping = False 268 done_looping = False
359 epoch = 0 269 epoch = 0
360 270 iter = 0
271
361 while (epoch < training_epochs) and (not done_looping): 272 while (epoch < training_epochs) and (not done_looping):
362 epoch = epoch + 1 273 epoch = epoch + 1
363 for minibatch_index in xrange(n_train_batches): 274 for x, y in dataset.train(batch_size):
364 275
365 cost_ij = network.finetune(minibatch_index) 276 cost_ij = network.finetune(x, y)
366 iter = epoch * n_train_batches + minibatch_index 277 iter += 1
367 278
368 if (iter+1) % validation_frequency == 0: 279 if iter % validation_frequency == 0:
369 280 validation_losses = [test_model(xv, yv) for xv, yv in dataset.valid(batch_size)]
370 validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
371 this_validation_loss = numpy.mean(validation_losses) 281 this_validation_loss = numpy.mean(validation_losses)
372 print('epoch %i, minibatch %i/%i, validation error %f %%' % \ 282 print('epoch %i, iter %i, validation error %f %%' % \
373 (epoch, minibatch_index+1, n_train_batches, \ 283 (epoch, iter, this_validation_loss*100.))
374 this_validation_loss*100.)) 284
375
376
377 # if we got the best validation score until now 285 # if we got the best validation score until now
378 if this_validation_loss < best_validation_loss: 286 if this_validation_loss < best_validation_loss:
379 287
380 #improve patience if loss improvement is good enough 288 #improve patience if loss improvement is good enough
381 if this_validation_loss < best_validation_loss * \ 289 if this_validation_loss < best_validation_loss * \
382 improvement_threshold : 290 improvement_threshold :
383 patience = max(patience, iter * patience_increase) 291 patience = max(patience, iter * patience_increase)
384 292
385 # save best validation score and iteration number 293 # save best validation score and iteration number
386 best_validation_loss = this_validation_loss 294 best_validation_loss = this_validation_loss
387 best_iter = iter 295 best_iter = iter
388 296
389 # test it on the test set 297 # test it on the test set
390 test_losses = [test_model(i) for i in xrange(n_test_batches)] 298 test_losses = [test_model(xt, yt) for xt, yt in dataset.test(batch_size)]
391 test_score = numpy.mean(test_losses) 299 test_score = numpy.mean(test_losses)
392 print((' epoch %i, minibatch %i/%i, test error of best ' 300 print((' epoch %i, iter %i, test error of best '
393 'model %f %%') % 301 'model %f %%') %
394 (epoch, minibatch_index+1, n_train_batches, 302 (epoch, iter, test_score*100.))
395 test_score*100.)) 303
396
397
398 if patience <= iter : 304 if patience <= iter :
399 done_looping = True 305 done_looping = True
400 break 306 break
401 307
402 end_time = time.clock() 308 end_time = time.clock()
403 print(('Optimization complete with best validation score of %f %%,' 309 print(('Optimization complete with best validation score of %f %%,'
404 'with test performance %f %%') % 310 'with test performance %f %%') %
405 (best_validation_loss * 100., test_score*100.)) 311 (best_validation_loss * 100., test_score*100.))
406 print ('The code ran for %f minutes' % ((end_time-start_time)/60.)) 312 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
407 313
408
409
410
411
412
413 if __name__ == '__main__': 314 if __name__ == '__main__':
414 sgd_optimization_mnist() 315 sgd_optimization_mnist()
415 316