Mercurial > ift6266

comparison deep/convolutional_dae/salah_exp/sgd_optimization_new.py @ 364:c05680f8c92f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixing a wrong commit and committing more files.
author humel
date Thu, 22 Apr 2010 19:50:21 -0400
parents
children
comparison
equal deleted inserted replaced
358:31641a84e0ae 364:c05680f8c92f
1 #!/usr/bin/python
2 # coding: utf-8
3
4 import numpy
5 import theano
6 import time
7 import datetime
8 import theano.tensor as T
9 import sys
10 import pickle
11
12 from jobman import DD
13 import jobman, jobman.sql
14 from copy import copy
15
16 from stacked_convolutional_dae_uit import CSdA
17
18 from ift6266.utils.seriestables import *
19
20 buffersize=1000
21
22 default_series = { \
23 'reconstruction_error' : DummySeries(),
24 'training_error' : DummySeries(),
25 'validation_error' : DummySeries(),
26 'test_error' : DummySeries(),
27 'params' : DummySeries()
28 }
29
30 def itermax(iter, max):
31 for i,it in enumerate(iter):
32 if i >= max:
33 break
34 yield it
35 def get_conv_shape(kernels,imgshp,batch_size,max_pool_layers):
36 # Returns the dimension at the output of the convoluational net
37 # and a list of Image and kernel shape for every
38 # Convolutional layer
39 conv_layers=[]
40 init_layer = [ [ kernels[0][0],1,kernels[0][1],kernels[0][2] ],\
41 [ batch_size , 1, imgshp[0], imgshp[1] ],
42 max_pool_layers[0] ]
43 conv_layers.append(init_layer)
44
45 conv_n_out = int((32-kernels[0][2]+1)/max_pool_layers[0][0])
46
47 for i in range(1,len(kernels)):
48 layer = [ [ kernels[i][0],kernels[i-1][0],kernels[i][1],kernels[i][2] ],\
49 [ batch_size, kernels[i-1][0],conv_n_out,conv_n_out ],
50 max_pool_layers[i] ]
51 conv_layers.append(layer)
52 conv_n_out = int( (conv_n_out - kernels[i][2]+1)/max_pool_layers[i][0])
53 conv_n_out=kernels[-1][0]*conv_n_out**2
54 return conv_n_out,conv_layers
55
56
57
58
59
60 class CSdASgdOptimizer:
61 def __init__(self, dataset, hyperparameters, n_ins, n_outs,
62 examples_per_epoch, series=default_series, max_minibatches=None):
63 self.dataset = dataset
64 self.hp = hyperparameters
65 self.n_ins = n_ins
66 self.n_outs = n_outs
67 self.parameters_pre=[]
68
69 self.max_minibatches = max_minibatches
70 print "CSdASgdOptimizer, max_minibatches =", max_minibatches
71
72 self.ex_per_epoch = examples_per_epoch
73 self.mb_per_epoch = examples_per_epoch / self.hp.minibatch_size
74
75 self.series = series
76
77 self.rng = numpy.random.RandomState(1234)
78 self.init_classifier()
79
80 sys.stdout.flush()
81
82 def init_classifier(self):
83 print "Constructing classifier"
84
85 n_ins,convlayers = get_conv_shape(self.hp.kernels,self.hp.imgshp,self.hp.minibatch_size,self.hp.max_pool_layers)
86
87 self.classifier = CSdA(n_ins_mlp = n_ins,
88 batch_size = self.hp.minibatch_size,
89 conv_hidden_layers_sizes = convlayers,
90 mlp_hidden_layers_sizes = self.hp.mlp_size,
91 corruption_levels = self.hp.corruption_levels,
92 rng = self.rng,
93 n_out = self.n_outs,
94 pretrain_lr = self.hp.pretraining_lr,
95 finetune_lr = self.hp.finetuning_lr)
96
97
98
99 #theano.printing.pydotprint(self.classifier.pretrain_functions[0], "function.graph")
100
101 sys.stdout.flush()
102
103 def train(self):
104 self.pretrain(self.dataset)
105 self.finetune(self.dataset)
106
107 def pretrain(self,dataset):
108 print "STARTING PRETRAINING, time = ", datetime.datetime.now()
109 sys.stdout.flush()
110
111 un_fichier=int(819200.0/self.hp.minibatch_size) #Number of batches in a P07 file
112
113 start_time = time.clock()
114 ## Pre-train layer-wise
115 for i in xrange(self.classifier.n_layers):
116 # go through pretraining epochs
117 for epoch in xrange(self.hp.pretraining_epochs_per_layer):
118 # go through the training set
119 batch_index=0
120 count=0
121 num_files=0
122 for x,y in dataset.train(self.hp.minibatch_size):
123 if x.shape[0] != self.hp.minibatch_size:
124 continue
125 c = self.classifier.pretrain_functions[i](x)
126 count +=1
127
128 self.series["reconstruction_error"].append((epoch, batch_index), c)
129 batch_index+=1
130
131 #if batch_index % 100 == 0:
132 # print "100 batches"
133
134 # useful when doing tests
135 if self.max_minibatches and batch_index >= self.max_minibatches:
136 break
137
138 #When we pass through the data only once (the case with P07)
139 #There is approximately 800*1024=819200 examples per file (1k per example and files are 800M)
140 if self.hp.pretraining_epochs_per_layer == 1 and count%un_fichier == 0:
141 print 'Pre-training layer %i, epoch %d, cost '%(i,num_files),c
142 num_files+=1
143 sys.stdout.flush()
144 self.series['params'].append((num_files,), self.classifier.all_params)
145
146 #When NIST is used
147 if self.hp.pretraining_epochs_per_layer > 1:
148 print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c
149 sys.stdout.flush()
150
151 self.series['params'].append((epoch,), self.classifier.all_params)
152 end_time = time.clock()
153
154 print ('Pretraining took %f minutes' %((end_time-start_time)/60.))
155 self.hp.update({'pretraining_time': end_time-start_time})
156
157 sys.stdout.flush()
158
159 #To be able to load them later for tests on finetune
160 self.parameters_pre=[copy(x.value) for x in self.classifier.params]
161 f = open('params_pretrain.txt', 'w')
162 pickle.dump(self.parameters_pre,f)
163 f.close()
164 def finetune(self,dataset,dataset_test,num_finetune,ind_test,special=0,decrease=0):
165
166 if special != 0 and special != 1:
167 sys.exit('Bad value for variable special. Must be in {0,1}')
168 print "STARTING FINETUNING, time = ", datetime.datetime.now()
169
170 minibatch_size = self.hp.minibatch_size
171 if ind_test == 0 or ind_test == 20:
172 nom_test = "NIST"
173 nom_train="P07"
174 else:
175 nom_test = "P07"
176 nom_train = "NIST"
177
178
179 # create a function to compute the mistakes that are made by the model
180 # on the validation set, or testing set
181 test_model = \
182 theano.function(
183 [self.classifier.x,self.classifier.y], self.classifier.errors)
184 # givens = {
185 # self.classifier.x: ensemble_x,
186 # self.classifier.y: ensemble_y]})
187
188 validate_model = \
189 theano.function(
190 [self.classifier.x,self.classifier.y], self.classifier.errors)
191 # givens = {
192 # self.classifier.x: ,
193 # self.classifier.y: ]})
194 # early-stopping parameters
195 patience = 10000 # look as this many examples regardless
196 patience_increase = 2. # wait this much longer when a new best is
197 # found
198 improvement_threshold = 0.995 # a relative improvement of this much is
199 # considered significant
200 validation_frequency = min(self.mb_per_epoch, patience/2)
201 # go through this many
202 # minibatche before checking the network
203 # on the validation set; in this case we
204 # check every epoch
205 if self.max_minibatches and validation_frequency > self.max_minibatches:
206 validation_frequency = self.max_minibatches / 2
207 best_params = None
208 best_validation_loss = float('inf')
209 test_score = 0.
210 start_time = time.clock()
211
212 done_looping = False
213 epoch = 0
214
215 total_mb_index = 0
216 minibatch_index = 0
217 parameters_finetune=[]
218 learning_rate = self.hp.finetuning_lr
219
220
221 while (epoch < num_finetune) and (not done_looping):
222 epoch = epoch + 1
223
224 for x,y in dataset.train(minibatch_size,bufsize=buffersize):
225
226 minibatch_index += 1
227
228 if x.shape[0] != self.hp.minibatch_size:
229 print 'bim'
230 continue
231
232 cost_ij = self.classifier.finetune(x,y)#,learning_rate)
233 total_mb_index += 1
234
235 self.series["training_error"].append((epoch, minibatch_index), cost_ij)
236
237 if (total_mb_index+1) % validation_frequency == 0:
238 #minibatch_index += 1
239 #The validation set is always NIST (we want the model to be good on NIST)
240
241 iter=dataset_test.valid(minibatch_size,bufsize=buffersize)
242
243
244 if self.max_minibatches:
245 iter = itermax(iter, self.max_minibatches)
246
247 validation_losses = []
248
249 for x,y in iter:
250 if x.shape[0] != self.hp.minibatch_size:
251 print 'bim'
252 continue
253 validation_losses.append(validate_model(x,y))
254
255 this_validation_loss = numpy.mean(validation_losses)
256
257 self.series["validation_error"].\
258 append((epoch, minibatch_index), this_validation_loss*100.)
259
260 print('epoch %i, minibatch %i, validation error on NIST : %f %%' % \
261 (epoch, minibatch_index+1, \
262 this_validation_loss*100.))
263
264
265 # if we got the best validation score until now
266 if this_validation_loss < best_validation_loss:
267
268 #improve patience if loss improvement is good enough
269 if this_validation_loss < best_validation_loss * \
270 improvement_threshold :
271 patience = max(patience, total_mb_index * patience_increase)
272
273 # save best validation score, iteration number and parameters
274 best_validation_loss = this_validation_loss
275 best_iter = total_mb_index
276 parameters_finetune=[copy(x.value) for x in self.classifier.params]
277
278 # test it on the test set
279 iter = dataset.test(minibatch_size,bufsize=buffersize)
280 if self.max_minibatches:
281 iter = itermax(iter, self.max_minibatches)
282 test_losses = []
283 test_losses2 = []
284 for x,y in iter:
285 if x.shape[0] != self.hp.minibatch_size:
286 print 'bim'
287 continue
288 test_losses.append(test_model(x,y))
289
290 test_score = numpy.mean(test_losses)
291
292 #test it on the second test set
293 iter2 = dataset_test.test(minibatch_size,bufsize=buffersize)
294 if self.max_minibatches:
295 iter2 = itermax(iter2, self.max_minibatches)
296 for x,y in iter2:
297 if x.shape[0] != self.hp.minibatch_size:
298 continue
299 test_losses2.append(test_model(x,y))
300
301 test_score2 = numpy.mean(test_losses2)
302
303 self.series["test_error"].\
304 append((epoch, minibatch_index), test_score*100.)
305
306 print((' epoch %i, minibatch %i, test error on dataset %s (train data) of best '
307 'model %f %%') %
308 (epoch, minibatch_index+1,nom_train,
309 test_score*100.))
310
311 print((' epoch %i, minibatch %i, test error on dataset %s of best '
312 'model %f %%') %
313 (epoch, minibatch_index+1,nom_test,
314 test_score2*100.))
315
316 if patience <= total_mb_index:
317 done_looping = True
318 break #to exit the FOR loop
319
320 sys.stdout.flush()
321
322 # useful when doing tests
323 if self.max_minibatches and minibatch_index >= self.max_minibatches:
324 break
325
326 if decrease == 1:
327 learning_rate /= 2 #divide the learning rate by 2 for each new epoch
328
329 self.series['params'].append((epoch,), self.classifier.all_params)
330
331 if done_looping == True: #To exit completly the fine-tuning
332 break #to exit the WHILE loop
333
334 end_time = time.clock()
335 self.hp.update({'finetuning_time':end_time-start_time,\
336 'best_validation_error':best_validation_loss,\
337 'test_score':test_score,
338 'num_finetuning_epochs':epoch})
339
340 print(('\nOptimization complete with best validation score of %f %%,'
341 'with test performance %f %% on dataset %s ') %
342 (best_validation_loss * 100., test_score*100.,nom_train))
343 print(('The test score on the %s dataset is %f')%(nom_test,test_score2*100.))
344
345 print ('The finetuning ran for %f minutes' % ((end_time-start_time)/60.))
346
347 sys.stdout.flush()
348
349 #Save a copy of the parameters in a file to be able to get them in the future
350
351 if special == 1: #To keep a track of the value of the parameters
352 f = open('params_finetune_stanford.txt', 'w')
353 pickle.dump(parameters_finetune,f)
354 f.close()
355
356 elif ind_test == 0 | ind_test == 20: #To keep a track of the value of the parameters
357 f = open('params_finetune_P07.txt', 'w')
358 pickle.dump(parameters_finetune,f)
359 f.close()
360
361
362 elif ind_test== 1: #For the run with 2 finetunes. It will be faster.
363 f = open('params_finetune_NIST.txt', 'w')
364 pickle.dump(parameters_finetune,f)
365 f.close()
366
367 elif ind_test== 21: #To keep a track of the value of the parameters
368 f = open('params_finetune_P07_then_NIST.txt', 'w')
369 pickle.dump(parameters_finetune,f)
370 f.close()
371 #Set parameters like they where right after pre-train or finetune
372 def reload_parameters(self,which):
373
374 #self.parameters_pre=pickle.load('params_pretrain.txt')
375 f = open(which)
376 self.parameters_pre=pickle.load(f)
377 f.close()
378 for idx,x in enumerate(self.parameters_pre):
379 if x.dtype=='float64':
380 self.classifier.params[idx].value=theano._asarray(copy(x),dtype=theano.config.floatX)
381 else:
382 self.classifier.params[idx].value=copy(x)
383
384 def training_error(self,dataset):
385 # create a function to compute the mistakes that are made by the model
386 # on the validation set, or testing set
387 test_model = \
388 theano.function(
389 [self.classifier.x,self.classifier.y], self.classifier.errors)
390
391 iter2 = dataset.train(self.hp.minibatch_size,bufsize=buffersize)
392 train_losses2 = [test_model(x,y) for x,y in iter2]
393 train_score2 = numpy.mean(train_losses2)
394 print "Training error is: " + str(train_score2)