Mercurial > ift6266

comparison deep/stacked_dae/v_sylvain/sgd_optimization.py @ 281:a8b92a4a708d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rajout de methode reliant toutes les couches cachees a la logistic et changeant seulement les parametres de la logistic durant finetune
author SylvainPL <sylvain.pannetier.lebeuf@umontreal.ca>
date Wed, 24 Mar 2010 14:44:41 -0400
parents a0264184684e
children 28b628f331b2
comparison
equal deleted inserted replaced
280:c77ffb11f91d 281:a8b92a4a708d
93 for i in xrange(self.classifier.n_layers): 93 for i in xrange(self.classifier.n_layers):
94 # go through pretraining epochs 94 # go through pretraining epochs
95 for epoch in xrange(self.hp.pretraining_epochs_per_layer): 95 for epoch in xrange(self.hp.pretraining_epochs_per_layer):
96 # go through the training set 96 # go through the training set
97 batch_index=0 97 batch_index=0
98 count=0
99 num_files=0
98 for x,y in dataset.train(self.hp.minibatch_size): 100 for x,y in dataset.train(self.hp.minibatch_size):
99 c = self.classifier.pretrain_functions[i](x) 101 c = self.classifier.pretrain_functions[i](x)
102 count +=1
100 103
101 self.series["reconstruction_error"].append((epoch, batch_index), c) 104 self.series["reconstruction_error"].append((epoch, batch_index), c)
102 batch_index+=1 105 batch_index+=1
103 106
104 #if batch_index % 100 == 0: 107 #if batch_index % 100 == 0:
105 # print "100 batches" 108 # print "100 batches"
106 109
107 # useful when doing tests 110 # useful when doing tests
108 if self.max_minibatches and batch_index >= self.max_minibatches: 111 if self.max_minibatches and batch_index >= self.max_minibatches:
109 break 112 break
110 113
111 print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c 114 #When we pass through the data only once (the case with P07)
112 sys.stdout.flush() 115 #There is approximately 800*1024=819200 examples per file (1k per example and files are 800M)
113 116 if self.hp.pretraining_epochs_per_layer == 1 and count%819200 == 0:
114 self.series['params'].append((epoch,), self.classifier.all_params) 117 print 'Pre-training layer %i, epoch %d, cost '%(i,num_files),c
118 num_files+=1
119 sys.stdout.flush()
120 self.series['params'].append((num_files,), self.classifier.all_params)
121
122 #When NIST is used
123 if self.hp.pretraining_epochs_per_layer > 1:
124 print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c
125 sys.stdout.flush()
126
127 self.series['params'].append((epoch,), self.classifier.all_params)
115 128
116 end_time = time.clock() 129 end_time = time.clock()
117 130
118 print ('Pretraining took %f minutes' %((end_time-start_time)/60.)) 131 print ('Pretraining took %f minutes' %((end_time-start_time)/60.))
119 self.hp.update({'pretraining_time': end_time-start_time}) 132 self.hp.update({'pretraining_time': end_time-start_time})
125 f = open('params_pretrain.txt', 'w') 138 f = open('params_pretrain.txt', 'w')
126 pickle.dump(self.parameters_pre,f) 139 pickle.dump(self.parameters_pre,f)
127 f.close() 140 f.close()
128 141
129 142
130 def finetune(self,dataset,dataset_test,num_finetune,ind_test): 143 def finetune(self,dataset,dataset_test,num_finetune,ind_test,special=0):
144
145 if special != 0 and special != 1:
146 sys.exit('Bad value for variable special. Must be in {0,1}')
131 print "STARTING FINETUNING, time = ", datetime.datetime.now() 147 print "STARTING FINETUNING, time = ", datetime.datetime.now()
132 148
133 minibatch_size = self.hp.minibatch_size 149 minibatch_size = self.hp.minibatch_size
134 if ind_test == 0: 150 if ind_test == 0 or ind_test == 20:
135 nom_test = "NIST" 151 nom_test = "NIST"
152 nom_train="P07"
136 else: 153 else:
137 nom_test = "P07" 154 nom_test = "P07"
155 nom_train = "NIST"
138 156
139 157
140 # create a function to compute the mistakes that are made by the model 158 # create a function to compute the mistakes that are made by the model
141 # on the validation set, or testing set 159 # on the validation set, or testing set
142 test_model = \ 160 test_model = \
181 while (epoch < num_finetune) and (not done_looping): 199 while (epoch < num_finetune) and (not done_looping):
182 epoch = epoch + 1 200 epoch = epoch + 1
183 minibatch_index = -1 201 minibatch_index = -1
184 for x,y in dataset.train(minibatch_size): 202 for x,y in dataset.train(minibatch_size):
185 minibatch_index += 1 203 minibatch_index += 1
186 cost_ij = self.classifier.finetune(x,y) 204 if special == 0:
205 cost_ij = self.classifier.finetune(x,y)
206 elif special == 1:
207 cost_ij = self.classifier.finetune2(x,y)
187 total_mb_index += 1 208 total_mb_index += 1
188 209
189 self.series["training_error"].append((epoch, minibatch_index), cost_ij) 210 self.series["training_error"].append((epoch, minibatch_index), cost_ij)
190 211
191 if (total_mb_index+1) % validation_frequency == 0: 212 if (total_mb_index+1) % validation_frequency == 0:
192 213
193 iter = dataset.valid(minibatch_size) 214 #The validation set is always NIST
215 if ind_test == 0:
216 iter=dataset_test.valid(minibatch_size)
217 else:
218 iter = dataset.valid(minibatch_size)
194 if self.max_minibatches: 219 if self.max_minibatches:
195 iter = itermax(iter, self.max_minibatches) 220 iter = itermax(iter, self.max_minibatches)
196 validation_losses = [validate_model(x,y) for x,y in iter] 221 validation_losses = [validate_model(x,y) for x,y in iter]
197 this_validation_loss = numpy.mean(validation_losses) 222 this_validation_loss = numpy.mean(validation_losses)
198 223
199 self.series["validation_error"].\ 224 self.series["validation_error"].\
200 append((epoch, minibatch_index), this_validation_loss*100.) 225 append((epoch, minibatch_index), this_validation_loss*100.)
201 226
202 print('epoch %i, minibatch %i, validation error %f %%' % \ 227 print('epoch %i, minibatch %i, validation error on %s : %f %%' % \
203 (epoch, minibatch_index+1, \ 228 (epoch, minibatch_index+1,nom_test, \
204 this_validation_loss*100.)) 229 this_validation_loss*100.))
205 230
206 231
207 # if we got the best validation score until now 232 # if we got the best validation score until now
208 if this_validation_loss < best_validation_loss: 233 if this_validation_loss < best_validation_loss:
231 test_score2 = numpy.mean(test_losses2) 256 test_score2 = numpy.mean(test_losses2)
232 257
233 self.series["test_error"].\ 258 self.series["test_error"].\
234 append((epoch, minibatch_index), test_score*100.) 259 append((epoch, minibatch_index), test_score*100.)
235 260
236 print((' epoch %i, minibatch %i, test error of best ' 261 print((' epoch %i, minibatch %i, test error on dataset %s (train data) of best '
237 'model %f %%') % 262 'model %f %%') %
238 (epoch, minibatch_index+1, 263 (epoch, minibatch_index+1,nom_train,
239 test_score*100.)) 264 test_score*100.))
240 265
241 print((' epoch %i, minibatch %i, test error on dataset %s of best ' 266 print((' epoch %i, minibatch %i, test error on dataset %s of best '
242 'model %f %%') % 267 'model %f %%') %
243 (epoch, minibatch_index+1,nom_test, 268 (epoch, minibatch_index+1,nom_test,
244 test_score2*100.)) 269 test_score2*100.))
245 270
271 if patience <= total_mb_index:
272 done_looping = True
273 break
274
246 sys.stdout.flush() 275 sys.stdout.flush()
247 276
248 # useful when doing tests 277 # useful when doing tests
249 if self.max_minibatches and minibatch_index >= self.max_minibatches: 278 if self.max_minibatches and minibatch_index >= self.max_minibatches:
250 break 279 break
251 280
252 self.series['params'].append((epoch,), self.classifier.all_params) 281 self.series['params'].append((epoch,), self.classifier.all_params)
253 282
254 if patience <= total_mb_index: 283 if done_looping == True: #To exit completly the fine-tuning
255 done_looping = True
256 break 284 break
257 285
258 end_time = time.clock() 286 end_time = time.clock()
259 self.hp.update({'finetuning_time':end_time-start_time,\ 287 self.hp.update({'finetuning_time':end_time-start_time,\
260 'best_validation_error':best_validation_loss,\ 288 'best_validation_error':best_validation_loss,\
261 'test_score':test_score, 289 'test_score':test_score,
262 'num_finetuning_epochs':epoch}) 290 'num_finetuning_epochs':epoch})
263 291
264 print(('Optimization complete with best validation score of %f %%,' 292 print(('\nOptimization complete with best validation score of %f %%,'
265 'with test performance %f %%') % 293 'with test performance %f %% on dataset %s ') %
266 (best_validation_loss * 100., test_score*100.)) 294 (best_validation_loss * 100., test_score*100.,nom_train))
267 print(('The test score on the %s dataset is %f')%(nom_test,test_score2*100.)) 295 print(('The test score on the %s dataset is %f')%(nom_test,test_score2*100.))
268 296
269 print ('The finetuning ran for %f minutes' % ((end_time-start_time)/60.)) 297 print ('The finetuning ran for %f minutes' % ((end_time-start_time)/60.))
270 298
299 #Save a copy of the parameters in a file to be able to get them in the future
300
301 if special == 1: #To keep a track of the value of the parameters
302 parameters_finetune=[copy(x.value) for x in self.classifier.params]
303 f = open('params_finetune_stanford.txt', 'w')
304 pickle.dump(parameters_finetune,f)
305 f.close()
306
307 elif ind_test== 0: #To keep a track of the value of the parameters
308 parameters_finetune=[copy(x.value) for x in self.classifier.params]
309 f = open('params_finetune_P07.txt', 'w')
310 pickle.dump(parameters_finetune,f)
311 f.close()
312
313 elif ind_test== 1: #For the run with 2 finetunes. It will be faster.
314 parameters_finetune=[copy(x.value) for x in self.classifier.params]
315 f = open('params_finetune_NIST.txt', 'w')
316 pickle.dump(parameters_finetune,f)
317 f.close()
318
319 elif ind_test== 20: #To keep a track of the value of the parameters
320 parameters_finetune=[copy(x.value) for x in self.classifier.params]
321 f = open('params_finetune_NIST_then_P07.txt', 'w')
322 pickle.dump(parameters_finetune,f)
323 f.close()
324
271 325
272 #Set parameters like they where right after pre-train 326 #Set parameters like they where right after pre-train
273 def reload_parameters(self): 327 def reload_parameters(self,which):
274 328
275 #self.parameters_pre=pickle.load('params_pretrain.txt') 329 #self.parameters_pre=pickle.load('params_pretrain.txt')
276 f = open('params_pretrain.txt') 330 f = open(which)
277 self.parameters_pre=pickle.load(f) 331 self.parameters_pre=pickle.load(f)
278 f.close() 332 f.close()
279 for idx,x in enumerate(self.parameters_pre): 333 for idx,x in enumerate(self.parameters_pre):
280 self.classifier.params[idx].value=copy(x) 334 self.classifier.params[idx].value=copy(x)
281 335