Mercurial > ift6266

comparison deep/stacked_dae/v_sylvain/sgd_optimization.py @ 230:8a94a5c808cd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Repertoire pour faire les tests avec les differents ensembles pour le finetuning
author SylvainPL <sylvain.pannetier.lebeuf@umontreal.ca>
date Fri, 12 Mar 2010 16:47:10 -0500
parents
children 02ed13244133
comparison
equal deleted inserted replaced
229:02eb98d051fe 230:8a94a5c808cd
1 #!/usr/bin/python
2 # coding: utf-8
3
4 # Generic SdA optimization loop, adapted from the deeplearning.net tutorial
5
6 import numpy
7 import theano
8 import time
9 import datetime
10 import theano.tensor as T
11 import sys
12
13 from jobman import DD
14 import jobman, jobman.sql
15
16 from stacked_dae import SdA
17
18 from ift6266.utils.seriestables import *
19
20 ##def shared_dataset(data_xy):
21 ## data_x, data_y = data_xy
22 ## if theano.config.device.startswith("gpu"):
23 ## print "TRANSFERING DATASETS (via shared()) TO GPU"
24 ## shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX))
25 ## shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX))
26 ## shared_y = T.cast(shared_y, 'int32')
27 ## else:
28 ## print "WILL RUN ON CPU, NOT GPU, SO DATASETS REMAIN IN BYTES"
29 ## shared_x = theano.shared(data_x)
30 ## shared_y = theano.shared(data_y)
31 ## return shared_x, shared_y
32
33 ######Les shared seront remplacees utilisant "given" dans les enonces de fonction plus loin
34 def shared_dataset(batch_size, n_in):
35
36 shared_x = theano.shared(numpy.asarray(numpy.zeros((batch_size,n_in)), dtype=theano.config.floatX))
37 shared_y = theano.shared(numpy.asarray(numpy.zeros(batch_size), dtype=theano.config.floatX))
38 return shared_x, shared_y
39
40 default_series = { \
41 'reconstruction_error' : DummySeries(),
42 'training_error' : DummySeries(),
43 'validation_error' : DummySeries(),
44 'test_error' : DummySeries(),
45 'params' : DummySeries()
46 }
47
48 class SdaSgdOptimizer:
49 def __init__(self, dataset, hyperparameters, n_ins, n_outs, input_divider=1.0, series=default_series):
50 self.dataset = dataset
51 self.hp = hyperparameters
52 self.n_ins = n_ins
53 self.n_outs = n_outs
54 self.input_divider = input_divider
55
56 self.series = series
57
58 self.rng = numpy.random.RandomState(1234)
59
60 self.init_datasets()
61 self.init_classifier()
62
63 sys.stdout.flush()
64
65 def init_datasets(self):
66 print "init_datasets"
67 sys.stdout.flush()
68
69 #train_set, valid_set, test_set = self.dataset
70 self.test_set_x, self.test_set_y = shared_dataset(self.hp.minibatch_size,self.n_ins)
71 self.valid_set_x, self.valid_set_y = shared_dataset(self.hp.minibatch_size,self.n_ins)
72 self.train_set_x, self.train_set_y = shared_dataset(self.hp.minibatch_size,self.n_ins)
73
74 # compute number of minibatches for training, validation and testing
75 self.n_train_batches = self.train_set_x.value.shape[0] / self.hp.minibatch_size
76 self.n_valid_batches = self.valid_set_x.value.shape[0] / self.hp.minibatch_size
77 # remove last batch in case it's incomplete
78 self.n_test_batches = (self.test_set_x.value.shape[0] / self.hp.minibatch_size) - 1
79
80 def init_classifier(self):
81 print "Constructing classifier"
82
83 # we don't want to save arrays in DD objects, so
84 # we recreate those arrays here
85 nhl = self.hp.num_hidden_layers
86 layers_sizes = [self.hp.hidden_layers_sizes] * nhl
87 corruption_levels = [self.hp.corruption_levels] * nhl
88
89 # construct the stacked denoising autoencoder class
90 self.classifier = SdA( \
91 train_set_x= self.train_set_x, \
92 train_set_y = self.train_set_y,\
93 batch_size = self.hp.minibatch_size, \
94 n_ins= self.n_ins, \
95 hidden_layers_sizes = layers_sizes, \
96 n_outs = self.n_outs, \
97 corruption_levels = corruption_levels,\
98 rng = self.rng,\
99 pretrain_lr = self.hp.pretraining_lr, \
100 finetune_lr = self.hp.finetuning_lr,\
101 input_divider = self.input_divider )
102
103 #theano.printing.pydotprint(self.classifier.pretrain_functions[0], "function.graph")
104
105 sys.stdout.flush()
106
107 def train(self):
108 self.pretrain()
109 self.finetune()
110
111 def pretrain(self):
112 print "STARTING PRETRAINING, time = ", datetime.datetime.now()
113 sys.stdout.flush()
114
115 start_time = time.clock()
116 ## Pre-train layer-wise
117 for i in xrange(self.classifier.n_layers):
118 # go through pretraining epochs
119 for epoch in xrange(self.hp.pretraining_epochs_per_layer):
120 # go through the training set
121 for batch_index in xrange(self.n_train_batches):
122 c = self.classifier.pretrain_functions[i](batch_index)
123
124 self.series["reconstruction_error"].append((epoch, batch_index), c)
125
126 print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c
127 sys.stdout.flush()
128
129 self.series['params'].append((epoch,), self.classifier.all_params)
130
131 end_time = time.clock()
132
133 print ('Pretraining took %f minutes' %((end_time-start_time)/60.))
134 self.hp.update({'pretraining_time': end_time-start_time})
135
136 sys.stdout.flush()
137
138 def finetune(self):
139 print "STARTING FINETUNING, time = ", datetime.datetime.now()
140
141 index = T.lscalar() # index to a [mini]batch
142 minibatch_size = self.hp.minibatch_size
143
144 # create a function to compute the mistakes that are made by the model
145 # on the validation set, or testing set
146 shared_divider = theano.shared(numpy.asarray(self.input_divider, dtype=theano.config.floatX))
147 test_model = theano.function([index], self.classifier.errors,
148 givens = {
149 self.classifier.x: self.test_set_x[index*minibatch_size:(index+1)*minibatch_size] / shared_divider,
150 self.classifier.y: self.test_set_y[index*minibatch_size:(index+1)*minibatch_size]})
151
152 validate_model = theano.function([index], self.classifier.errors,
153 givens = {
154 self.classifier.x: self.valid_set_x[index*minibatch_size:(index+1)*minibatch_size] / shared_divider,
155 self.classifier.y: self.valid_set_y[index*minibatch_size:(index+1)*minibatch_size]})
156
157
158 # early-stopping parameters
159 patience = 10000 # look as this many examples regardless
160 patience_increase = 2. # wait this much longer when a new best is
161 # found
162 improvement_threshold = 0.995 # a relative improvement of this much is
163 # considered significant
164 validation_frequency = min(self.n_train_batches, patience/2)
165 # go through this many
166 # minibatche before checking the network
167 # on the validation set; in this case we
168 # check every epoch
169
170 best_params = None
171 best_validation_loss = float('inf')
172 test_score = 0.
173 start_time = time.clock()
174
175 done_looping = False
176 epoch = 0
177
178 while (epoch < self.hp.max_finetuning_epochs) and (not done_looping):
179 epoch = epoch + 1
180 for minibatch_index in xrange(self.n_train_batches):
181
182 cost_ij = self.classifier.finetune(minibatch_index)
183 iter = epoch * self.n_train_batches + minibatch_index
184
185 self.series["training_error"].append((epoch, minibatch_index), cost_ij)
186
187 if (iter+1) % validation_frequency == 0:
188
189 validation_losses = [validate_model(i) for i in xrange(self.n_valid_batches)]
190 this_validation_loss = numpy.mean(validation_losses)
191
192 self.series["validation_error"].\
193 append((epoch, minibatch_index), this_validation_loss*100.)
194
195 print('epoch %i, minibatch %i/%i, validation error %f %%' % \
196 (epoch, minibatch_index+1, self.n_train_batches, \
197 this_validation_loss*100.))
198
199
200 # if we got the best validation score until now
201 if this_validation_loss < best_validation_loss:
202
203 #improve patience if loss improvement is good enough
204 if this_validation_loss < best_validation_loss * \
205 improvement_threshold :
206 patience = max(patience, iter * patience_increase)
207
208 # save best validation score and iteration number
209 best_validation_loss = this_validation_loss
210 best_iter = iter
211
212 # test it on the test set
213 test_losses = [test_model(i) for i in xrange(self.n_test_batches)]
214 test_score = numpy.mean(test_losses)
215
216 self.series["test_error"].\
217 append((epoch, minibatch_index), test_score*100.)
218
219 print((' epoch %i, minibatch %i/%i, test error of best '
220 'model %f %%') %
221 (epoch, minibatch_index+1, self.n_train_batches,
222 test_score*100.))
223
224 sys.stdout.flush()
225
226 self.series['params'].append((epoch,), self.classifier.all_params)
227
228 if patience <= iter :
229 done_looping = True
230 break
231
232 end_time = time.clock()
233 self.hp.update({'finetuning_time':end_time-start_time,\
234 'best_validation_error':best_validation_loss,\
235 'test_score':test_score,
236 'num_finetuning_epochs':epoch})
237
238 print(('Optimization complete with best validation score of %f %%,'
239 'with test performance %f %%') %
240 (best_validation_loss * 100., test_score*100.))
241 print ('The finetuning ran for %f minutes' % ((end_time-start_time)/60.))
242
243
244