Mercurial > ift6266

comparison deep/stacked_dae/v2/sgd_optimization.py @ 227:acae439d6572

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Ajouté une modification sur stacked_dae qui utilise les nouvelles SeriesTables. Je le met dans le repository pour que mes expériences en cours continuent sans perturbation, et pour que Sylvain puisse récupérer la version actuelle; je fusionnerai à moment donné.
author fsavard
date Fri, 12 Mar 2010 10:31:10 -0500
parents
children 851e7ad4a143
comparison
equal deleted inserted replaced
226:bfe20d63f88c 227:acae439d6572
1 #!/usr/bin/python
2 # coding: utf-8
3
4 # Generic SdA optimization loop, adapted from the deeplearning.net tutorial
5
6 import numpy
7 import theano
8 import time
9 import datetime
10 import theano.tensor as T
11 import sys
12
13 from jobman import DD
14 import jobman, jobman.sql
15
16 from stacked_dae import SdA
17
18 from ift6266.utils.seriestables import *
19
20 def shared_dataset(data_xy):
21 data_x, data_y = data_xy
22 if theano.config.device.startswith("gpu"):
23 print "TRANSFERING DATASETS (via shared()) TO GPU"
24 shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX))
25 shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX))
26 shared_y = T.cast(shared_y, 'int32')
27 else:
28 shared_x = theano.shared(data_x)
29 shared_y = theano.shared(data_y)
30 return shared_x, shared_y
31
32 default_series = { \
33 'reconstruction_error' : DummySeries(),
34 'training_error' : DummySeries(),
35 'validation_error' : DummySeries(),
36 'test_error' : DummySeries(),
37 'params' : DummySeries()
38 }
39
40 class SdaSgdOptimizer:
41 def __init__(self, dataset, hyperparameters, n_ins, n_outs, input_divider=1.0, series=default_series):
42 self.dataset = dataset
43 self.hp = hyperparameters
44 self.n_ins = n_ins
45 self.n_outs = n_outs
46 self.input_divider = input_divider
47
48 self.series = series
49
50 self.rng = numpy.random.RandomState(1234)
51
52 self.init_datasets()
53 self.init_classifier()
54
55 sys.stdout.flush()
56
57 def init_datasets(self):
58 print "init_datasets"
59 sys.stdout.flush()
60
61 train_set, valid_set, test_set = self.dataset
62 self.test_set_x, self.test_set_y = shared_dataset(test_set)
63 self.valid_set_x, self.valid_set_y = shared_dataset(valid_set)
64 self.train_set_x, self.train_set_y = shared_dataset(train_set)
65
66 # compute number of minibatches for training, validation and testing
67 self.n_train_batches = self.train_set_x.value.shape[0] / self.hp.minibatch_size
68 self.n_valid_batches = self.valid_set_x.value.shape[0] / self.hp.minibatch_size
69 # remove last batch in case it's incomplete
70 self.n_test_batches = (self.test_set_x.value.shape[0] / self.hp.minibatch_size) - 1
71
72 def init_classifier(self):
73 print "Constructing classifier"
74
75 # we don't want to save arrays in DD objects, so
76 # we recreate those arrays here
77 nhl = self.hp.num_hidden_layers
78 layers_sizes = [self.hp.hidden_layers_sizes] * nhl
79 corruption_levels = [self.hp.corruption_levels] * nhl
80
81 # construct the stacked denoising autoencoder class
82 self.classifier = SdA( \
83 train_set_x= self.train_set_x, \
84 train_set_y = self.train_set_y,\
85 batch_size = self.hp.minibatch_size, \
86 n_ins= self.n_ins, \
87 hidden_layers_sizes = layers_sizes, \
88 n_outs = self.n_outs, \
89 corruption_levels = corruption_levels,\
90 rng = self.rng,\
91 pretrain_lr = self.hp.pretraining_lr, \
92 finetune_lr = self.hp.finetuning_lr,\
93 input_divider = self.input_divider )
94
95 #theano.printing.pydotprint(self.classifier.pretrain_functions[0], "function.graph")
96
97 sys.stdout.flush()
98
99 def train(self):
100 self.pretrain()
101 self.finetune()
102
103 def pretrain(self):
104 print "STARTING PRETRAINING, time = ", datetime.datetime.now()
105 sys.stdout.flush()
106
107 time_acc_func = 0.0
108 time_acc_total = 0.0
109
110 start_time = time.clock()
111 ## Pre-train layer-wise
112 for i in xrange(self.classifier.n_layers):
113 # go through pretraining epochs
114 for epoch in xrange(self.hp.pretraining_epochs_per_layer):
115 # go through the training set
116 for batch_index in xrange(self.n_train_batches):
117 t1 = time.clock()
118 c = self.classifier.pretrain_functions[i](batch_index)
119 t2 = time.clock()
120
121 time_acc_func += t2 - t1
122
123 if batch_index % 500 == 0:
124 print "acc / total", time_acc_func / (t2 - start_time), time_acc_func
125
126 self.series["reconstruction_error"].append((epoch, batch_index), c)
127
128 print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c
129 sys.stdout.flush()
130
131 self.series['params'].append((epoch,), self.classifier.all_params)
132
133 end_time = time.clock()
134
135 print ('Pretraining took %f minutes' %((end_time-start_time)/60.))
136 self.hp.update({'pretraining_time': end_time-start_time})
137
138 sys.stdout.flush()
139
140 def finetune(self):
141 print "STARTING FINETUNING, time = ", datetime.datetime.now()
142
143 index = T.lscalar() # index to a [mini]batch
144 minibatch_size = self.hp.minibatch_size
145
146 # create a function to compute the mistakes that are made by the model
147 # on the validation set, or testing set
148 shared_divider = theano.shared(numpy.asarray(self.input_divider, dtype=theano.config.floatX))
149 test_model = theano.function([index], self.classifier.errors,
150 givens = {
151 self.classifier.x: self.test_set_x[index*minibatch_size:(index+1)*minibatch_size] / shared_divider,
152 self.classifier.y: self.test_set_y[index*minibatch_size:(index+1)*minibatch_size]})
153
154 validate_model = theano.function([index], self.classifier.errors,
155 givens = {
156 self.classifier.x: self.valid_set_x[index*minibatch_size:(index+1)*minibatch_size] / shared_divider,
157 self.classifier.y: self.valid_set_y[index*minibatch_size:(index+1)*minibatch_size]})
158
159
160 # early-stopping parameters
161 patience = 10000 # look as this many examples regardless
162 patience_increase = 2. # wait this much longer when a new best is
163 # found
164 improvement_threshold = 0.995 # a relative improvement of this much is
165 # considered significant
166 validation_frequency = min(self.n_train_batches, patience/2)
167 # go through this many
168 # minibatche before checking the network
169 # on the validation set; in this case we
170 # check every epoch
171
172 best_params = None
173 best_validation_loss = float('inf')
174 test_score = 0.
175 start_time = time.clock()
176
177 done_looping = False
178 epoch = 0
179
180 while (epoch < self.hp.max_finetuning_epochs) and (not done_looping):
181 epoch = epoch + 1
182 for minibatch_index in xrange(self.n_train_batches):
183
184 cost_ij = self.classifier.finetune(minibatch_index)
185 iter = epoch * self.n_train_batches + minibatch_index
186
187 self.series["training_error"].append((epoch, minibatch_index), cost_ij)
188
189 if (iter+1) % validation_frequency == 0:
190
191 validation_losses = [validate_model(i) for i in xrange(self.n_valid_batches)]
192 this_validation_loss = numpy.mean(validation_losses)
193
194 self.series["validation_error"].\
195 append((epoch, minibatch_index), this_validation_loss*100.)
196
197 print('epoch %i, minibatch %i/%i, validation error %f %%' % \
198 (epoch, minibatch_index+1, self.n_train_batches, \
199 this_validation_loss*100.))
200
201
202 # if we got the best validation score until now
203 if this_validation_loss < best_validation_loss:
204
205 #improve patience if loss improvement is good enough
206 if this_validation_loss < best_validation_loss * \
207 improvement_threshold :
208 patience = max(patience, iter * patience_increase)
209
210 # save best validation score and iteration number
211 best_validation_loss = this_validation_loss
212 best_iter = iter
213
214 # test it on the test set
215 test_losses = [test_model(i) for i in xrange(self.n_test_batches)]
216 test_score = numpy.mean(test_losses)
217
218 self.series["test_error"].\
219 append((epoch, minibatch_index), test_score*100.)
220
221 print((' epoch %i, minibatch %i/%i, test error of best '
222 'model %f %%') %
223 (epoch, minibatch_index+1, self.n_train_batches,
224 test_score*100.))
225
226 sys.stdout.flush()
227
228 self.series['params'].append((epoch,), self.classifier.all_params)
229
230 if patience <= iter :
231 done_looping = True
232 break
233
234 end_time = time.clock()
235 self.hp.update({'finetuning_time':end_time-start_time,\
236 'best_validation_error':best_validation_loss,\
237 'test_score':test_score,
238 'num_finetuning_epochs':epoch})
239
240 print(('Optimization complete with best validation score of %f %%,'
241 'with test performance %f %%') %
242 (best_validation_loss * 100., test_score*100.))
243 print ('The finetuning ran for %f minutes' % ((end_time-start_time)/60.))
244
245
246