Mercurial > ift6266

comparison deep/stacked_dae/v_guillaume/sgd_optimization.py @ 436:0ca069550abd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added : single class version of SDA
author Guillaume Sicard <guitch21@gmail.com>
date Mon, 03 May 2010 06:14:05 -0400
parents
children
comparison
equal deleted inserted replaced
435:d8129a09ffb1 436:0ca069550abd
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 # coding: utf-8
4
5 # Generic SdA optimization loop, adapted from the deeplearning.net tutorial
6
7 import numpy
8 import theano
9 import time
10 import datetime
11 import theano.tensor as T
12 import sys
13 #import pickle
14 import cPickle
15
16 from jobman import DD
17 import jobman, jobman.sql
18 from copy import copy
19
20 from stacked_dae import SdA
21
22 from ift6266.utils.seriestables import *
23
24 #For test purpose only
25 buffersize=1000
26
27 default_series = { \
28 'reconstruction_error' : DummySeries(),
29 'training_error' : DummySeries(),
30 'validation_error' : DummySeries(),
31 'test_error' : DummySeries(),
32 'params' : DummySeries()
33 }
34
35 def itermax(iter, max):
36 for i,it in enumerate(iter):
37 if i >= max:
38 break
39 yield it
40
41 class SdaSgdOptimizer:
42 def __init__(self, dataset_name, dataset, hyperparameters, n_ins, n_outs,
43 examples_per_epoch, series=default_series, max_minibatches=None):
44 self.dataset_name = dataset_name
45 self.dataset = dataset
46 self.hp = hyperparameters
47 self.n_ins = n_ins
48 self.n_outs = n_outs
49 self.parameters_pre=[]
50
51 if (self.dataset_name == "upper"):
52 self.class_offset = 10
53 elif (self.dataset_name == "lower"):
54 self.class_offset = 36
55 else:
56 self.class_offset = 0
57
58
59 self.max_minibatches = max_minibatches
60 print "SdaSgdOptimizer, max_minibatches =", max_minibatches
61
62 self.ex_per_epoch = examples_per_epoch
63 self.mb_per_epoch = examples_per_epoch / self.hp.minibatch_size
64
65 self.series = series
66
67 self.rng = numpy.random.RandomState(1234)
68
69 self.init_classifier()
70
71 sys.stdout.flush()
72
73 def init_classifier(self):
74 print "Constructing classifier"
75
76 # we don't want to save arrays in DD objects, so
77 # we recreate those arrays here
78 nhl = self.hp.num_hidden_layers
79 layers_sizes = [self.hp.hidden_layers_sizes] * nhl
80 corruption_levels = [self.hp.corruption_levels] * nhl
81
82 # construct the stacked denoising autoencoder class
83 self.classifier = SdA( \
84 batch_size = self.hp.minibatch_size, \
85 n_ins= self.n_ins, \
86 hidden_layers_sizes = layers_sizes, \
87 n_outs = self.n_outs, \
88 corruption_levels = corruption_levels,\
89 rng = self.rng,\
90 pretrain_lr = self.hp.pretraining_lr, \
91 finetune_lr = self.hp.finetuning_lr)
92
93 #theano.printing.pydotprint(self.classifier.pretrain_functions[0], "function.graph")
94
95 sys.stdout.flush()
96
97 def train(self):
98 self.pretrain(self.dataset)
99 self.finetune(self.dataset)
100
101 def pretrain(self,dataset,decrease=0):
102 print "STARTING PRETRAINING, time = ", datetime.datetime.now()
103 sys.stdout.flush()
104
105 un_fichier=int(819200.0/self.hp.minibatch_size) #Number of batches in a P07 file
106
107 start_time = time.clock()
108
109 ######## This is hardcoaded. THe 0.95 parameter is hardcoaded and can be changed at will ###
110 #Set the decreasing rate of the learning rate. We want the final learning rate to
111 #be 5% of the original learning rate. The decreasing factor is linear
112 decreasing = (decrease*self.hp.pretraining_lr)/float(self.hp.pretraining_epochs_per_layer*800000/self.hp.minibatch_size)
113
114 ## Pre-train layer-wise
115 for i in xrange(self.classifier.n_layers):
116 # go through pretraining epochs
117
118 #To reset the learning rate to his original value
119 learning_rate=self.hp.pretraining_lr
120 for epoch in xrange(self.hp.pretraining_epochs_per_layer):
121 # go through the training set
122 batch_index=0
123 count=0
124 num_files=0
125 for x,y in dataset.train(self.hp.minibatch_size):
126 y = y - self.class_offset
127 c = self.classifier.pretrain_functions[i](x,learning_rate)
128 count +=1
129
130 self.series["reconstruction_error"].append((epoch, batch_index), c)
131 batch_index+=1
132
133 #If we need to decrease the learning rate for the pretrain
134 if decrease != 0:
135 learning_rate -= decreasing
136
137 # useful when doing tests
138 if self.max_minibatches and batch_index >= self.max_minibatches:
139 break
140
141 #When we pass through the data only once (the case with P07)
142 #There is approximately 800*1024=819200 examples per file (1k per example and files are 800M)
143 if self.hp.pretraining_epochs_per_layer == 1 and count%un_fichier == 0:
144 print 'Pre-training layer %i, epoch %d, cost '%(i,num_files),c
145 num_files+=1
146 sys.stdout.flush()
147 self.series['params'].append((num_files,), self.classifier.all_params)
148
149 #When NIST is used
150 if self.hp.pretraining_epochs_per_layer > 1:
151 print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c
152 sys.stdout.flush()
153
154 self.series['params'].append((epoch,), self.classifier.all_params)
155
156 end_time = time.clock()
157
158 print ('Pretraining took %f minutes' %((end_time-start_time)/60.))
159 self.hp.update({'pretraining_time': end_time-start_time})
160
161 sys.stdout.flush()
162
163 #To be able to load them later for tests on finetune
164 self.parameters_pre=[copy(x.value) for x in self.classifier.params]
165 f = open('params_pretrain.txt', 'w')
166 cPickle.dump(self.parameters_pre,f,protocol=-1)
167 f.close()
168
169
170 def finetune(self,dataset,dataset_test,num_finetune,ind_test,special=0,decrease=0):
171
172 if special != 0 and special != 1:
173 sys.exit('Bad value for variable special. Must be in {0,1}')
174 print "STARTING FINETUNING, time = ", datetime.datetime.now()
175
176 minibatch_size = self.hp.minibatch_size
177 if ind_test == 0 or ind_test == 20:
178 nom_test = "NIST"
179 nom_train="P07"
180 else:
181 nom_test = "P07"
182 nom_train = "NIST"
183
184
185 # create a function to compute the mistakes that are made by the model
186 # on the validation set, or testing set
187 test_model = \
188 theano.function(
189 [self.classifier.x,self.classifier.y], self.classifier.errors)
190 # givens = {
191 # self.classifier.x: ensemble_x,
192 # self.classifier.y: ensemble_y]})
193
194 validate_model = \
195 theano.function(
196 [self.classifier.x,self.classifier.y], self.classifier.errors)
197 # givens = {
198 # self.classifier.x: ,
199 # self.classifier.y: ]})
200
201
202 # early-stopping parameters
203 patience = 10000 # look as this many examples regardless
204 patience_increase = 2. # wait this much longer when a new best is
205 # found
206 improvement_threshold = 0.995 # a relative improvement of this much is
207 # considered significant
208 validation_frequency = min(self.mb_per_epoch, patience/2)
209 # go through this many
210 # minibatche before checking the network
211 # on the validation set; in this case we
212 # check every epoch
213 if self.max_minibatches and validation_frequency > self.max_minibatches:
214 validation_frequency = self.max_minibatches / 2
215
216 best_params = None
217 best_validation_loss = float('inf')
218 test_score = 0.
219 start_time = time.clock()
220
221 done_looping = False
222 epoch = 0
223
224 total_mb_index = 0
225 minibatch_index = 0
226 parameters_finetune=[]
227
228 if ind_test == 21:
229 learning_rate = self.hp.finetuning_lr / 10.0
230 else:
231 learning_rate = self.hp.finetuning_lr #The initial finetune lr
232
233
234 while (epoch < num_finetune) and (not done_looping):
235 epoch = epoch + 1
236
237 for x,y in dataset.train(minibatch_size,bufsize=buffersize):
238 minibatch_index += 1
239
240 y = y - self.class_offset
241
242 if special == 0:
243 cost_ij = self.classifier.finetune(x,y,learning_rate)
244 elif special == 1:
245 cost_ij = self.classifier.finetune2(x,y)
246 total_mb_index += 1
247
248 self.series["training_error"].append((epoch, minibatch_index), cost_ij)
249
250 if (total_mb_index+1) % validation_frequency == 0:
251 #minibatch_index += 1
252 #The validation set is always NIST (we want the model to be good on NIST)
253 if ind_test == 0 | ind_test == 20:
254 iter=dataset_test.valid(minibatch_size,bufsize=buffersize)
255 else:
256 iter = dataset.valid(minibatch_size,bufsize=buffersize)
257 if self.max_minibatches:
258 iter = itermax(iter, self.max_minibatches)
259 validation_losses = [validate_model(x,y - self.class_offset) for x,y in iter]
260 this_validation_loss = numpy.mean(validation_losses)
261
262 self.series["validation_error"].\
263 append((epoch, minibatch_index), this_validation_loss*100.)
264
265 print('epoch %i, minibatch %i, validation error on NIST : %f %%' % \
266 (epoch, minibatch_index+1, \
267 this_validation_loss*100.))
268
269
270 # if we got the best validation score until now
271 if this_validation_loss < best_validation_loss:
272
273 #improve patience if loss improvement is good enough
274 if this_validation_loss < best_validation_loss * \
275 improvement_threshold :
276 patience = max(patience, total_mb_index * patience_increase)
277
278 # save best validation score, iteration number and parameters
279 best_validation_loss = this_validation_loss
280 best_iter = total_mb_index
281 parameters_finetune=[copy(x.value) for x in self.classifier.params]
282
283 # test it on the test set
284 iter = dataset.test(minibatch_size,bufsize=buffersize)
285 if self.max_minibatches:
286 iter = itermax(iter, self.max_minibatches)
287 test_losses = [test_model(x,y - self.class_offset) for x,y in iter]
288 test_score = numpy.mean(test_losses)
289
290 #test it on the second test set
291 iter2 = dataset_test.test(minibatch_size,bufsize=buffersize)
292 if self.max_minibatches:
293 iter2 = itermax(iter2, self.max_minibatches)
294 test_losses2 = [test_model(x,y - self.class_offset) for x,y in iter2]
295 test_score2 = numpy.mean(test_losses2)
296
297 self.series["test_error"].\
298 append((epoch, minibatch_index), test_score*100.)
299
300 print((' epoch %i, minibatch %i, test error on dataset %s (train data) of best '
301 'model %f %%') %
302 (epoch, minibatch_index+1,nom_train,
303 test_score*100.))
304
305 print((' epoch %i, minibatch %i, test error on dataset %s of best '
306 'model %f %%') %
307 (epoch, minibatch_index+1,nom_test,
308 test_score2*100.))
309
310 if patience <= total_mb_index:
311 done_looping = True
312 break #to exit the FOR loop
313
314 sys.stdout.flush()
315
316 # useful when doing tests
317 if self.max_minibatches and minibatch_index >= self.max_minibatches:
318 break
319
320 if decrease == 1:
321 if (ind_test == 21 & epoch % 100 == 0) | ind_test == 20 | (ind_test == 1 & epoch % 100 == 0) :
322 learning_rate /= 2 #divide the learning rate by 2 for each new epoch of P07 (or 100 of NIST)
323
324 self.series['params'].append((epoch,), self.classifier.all_params)
325
326 if done_looping == True: #To exit completly the fine-tuning
327 break #to exit the WHILE loop
328
329 end_time = time.clock()
330 self.hp.update({'finetuning_time':end_time-start_time,\
331 'best_validation_error':best_validation_loss,\
332 'test_score':test_score,
333 'num_finetuning_epochs':epoch})
334
335 print(('\nOptimization complete with best validation score of %f %%,'
336 'with test performance %f %% on dataset %s ') %
337 (best_validation_loss * 100., test_score*100.,nom_train))
338 print(('The test score on the %s dataset is %f')%(nom_test,test_score2*100.))
339
340 print ('The finetuning ran for %f minutes' % ((end_time-start_time)/60.))
341
342 sys.stdout.flush()
343
344 #Save a copy of the parameters in a file to be able to get them in the future
345
346 if special == 1: #To keep a track of the value of the parameters
347 f = open('params_finetune_stanford.txt', 'w')
348 cPickle.dump(parameters_finetune,f,protocol=-1)
349 f.close()
350
351 elif ind_test == 0 | ind_test == 20: #To keep a track of the value of the parameters
352 f = open('params_finetune_P07.txt', 'w')
353 cPickle.dump(parameters_finetune,f,protocol=-1)
354 f.close()
355
356
357 elif ind_test== 1: #For the run with 2 finetunes. It will be faster.
358 f = open('params_finetune_NIST.txt', 'w')
359 cPickle.dump(parameters_finetune,f,protocol=-1)
360 f.close()
361
362 elif ind_test== 21: #To keep a track of the value of the parameters
363 f = open('params_finetune_P07_then_NIST.txt', 'w')
364 cPickle.dump(parameters_finetune,f,protocol=-1)
365 f.close()
366
367
368 #Set parameters like they where right after pre-train or finetune
369 def reload_parameters(self,which):
370
371 #self.parameters_pre=pickle.load('params_pretrain.txt')
372 f = open(which)
373 self.parameters_pre=cPickle.load(f)
374 f.close()
375 for idx,x in enumerate(self.parameters_pre):
376 if x.dtype=='float64':
377 self.classifier.params[idx].value=theano._asarray(copy(x),dtype=theano.config.floatX)
378 else:
379 self.classifier.params[idx].value=copy(x)
380
381 def training_error(self,dataset):
382 # create a function to compute the mistakes that are made by the model
383 # on the validation set, or testing set
384 test_model = \
385 theano.function(
386 [self.classifier.x,self.classifier.y], self.classifier.errors)
387
388 iter2 = dataset.train(self.hp.minibatch_size,bufsize=buffersize)
389 train_losses2 = [test_model(x,y - self.class_offset) for x,y in iter2]
390 train_score2 = numpy.mean(train_losses2)
391 print "Training error is: " + str(train_score2)
392
393
394
395