Mercurial > ift6266

comparison scripts/stacked_dae/sgd_optimization.py @ 131:5c79a2557f2f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Un peu de ménage dans code pour stacked DAE, splitté en fichiers dans un nouveau sous-répertoire.
author savardf
date Fri, 19 Feb 2010 08:43:10 -0500
parents
children 7d8366fb90bf
comparison
equal deleted inserted replaced
130:38929c29b602 131:5c79a2557f2f
1 #!/usr/bin/python
2 # coding: utf-8
3
4 # Generic SdA optimization loop, adapted slightly from the deeplearning.net tutorial
5
6 import numpy
7 import theano
8 import time
9 import theano.tensor as T
10
11 from jobman import DD
12
13 from stacked_dae import SdA
14
15 def sgd_optimization(dataset, hyperparameters, n_ins, n_outs):
16 hp = hyperparameters
17
18 printout_frequency = 1000
19
20 train_set, valid_set, test_set = dataset
21
22 def shared_dataset(data_xy):
23 data_x, data_y = data_xy
24 shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX))
25 shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX))
26 return shared_x, T.cast(shared_y, 'int32')
27
28 test_set_x, test_set_y = shared_dataset(test_set)
29 valid_set_x, valid_set_y = shared_dataset(valid_set)
30 train_set_x, train_set_y = shared_dataset(train_set)
31
32 # compute number of minibatches for training, validation and testing
33 n_train_batches = train_set_x.value.shape[0] / hp.minibatch_size
34 n_valid_batches = valid_set_x.value.shape[0] / hp.minibatch_size
35 n_test_batches = test_set_x.value.shape[0] / hp.minibatch_size
36
37 # allocate symbolic variables for the data
38 index = T.lscalar() # index to a [mini]batch
39
40 # construct the stacked denoising autoencoder class
41 classifier = SdA( train_set_x=train_set_x, train_set_y = train_set_y,\
42 batch_size = hp.minibatch_size, n_ins= n_ins, \
43 hidden_layers_sizes = hp.hidden_layers_sizes, n_outs=10, \
44 corruption_levels = hp.corruption_levels,\
45 rng = numpy.random.RandomState(1234),\
46 pretrain_lr = hp.pretraining_lr, finetune_lr = hp.finetuning_lr )
47
48 printout_acc = 0.0
49
50 start_time = time.clock()
51 ## Pre-train layer-wise
52 for i in xrange(classifier.n_layers):
53 # go through pretraining epochs
54 for epoch in xrange(hp.pretraining_epochs_per_layer):
55 # go through the training set
56 for batch_index in xrange(n_train_batches):
57 c = classifier.pretrain_functions[i](batch_index)
58
59 print c
60
61 printout_acc += c / printout_frequency
62 if (batch_index+1) % printout_frequency == 0:
63 print batch_index, "reconstruction cost avg=", printout_acc
64 printout_acc = 0.0
65
66 print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c
67
68 end_time = time.clock()
69
70 print ('Pretraining took %f minutes' %((end_time-start_time)/60.))
71 # Fine-tune the entire model
72
73 minibatch_size = hp.minibatch_size
74
75 # create a function to compute the mistakes that are made by the model
76 # on the validation set, or testing set
77 test_model = theano.function([index], classifier.errors,
78 givens = {
79 classifier.x: test_set_x[index*minibatch_size:(index+1)*minibatch_size],
80 classifier.y: test_set_y[index*minibatch_size:(index+1)*minibatch_size]})
81
82 validate_model = theano.function([index], classifier.errors,
83 givens = {
84 classifier.x: valid_set_x[index*minibatch_size:(index+1)*minibatch_size],
85 classifier.y: valid_set_y[index*minibatch_size:(index+1)*minibatch_size]})
86
87
88 # early-stopping parameters
89 patience = 10000 # look as this many examples regardless
90 patience_increase = 2. # wait this much longer when a new best is
91 # found
92 improvement_threshold = 0.995 # a relative improvement of this much is
93 # considered significant
94 validation_frequency = min(n_train_batches, patience/2)
95 # go through this many
96 # minibatche before checking the network
97 # on the validation set; in this case we
98 # check every epoch
99
100 best_params = None
101 best_validation_loss = float('inf')
102 test_score = 0.
103 start_time = time.clock()
104
105 done_looping = False
106 epoch = 0
107
108 printout_acc = 0.0
109
110 print "----- START FINETUNING -----"
111
112 while (epoch < hp.max_finetuning_epochs) and (not done_looping):
113 epoch = epoch + 1
114 for minibatch_index in xrange(n_train_batches):
115
116 cost_ij = classifier.finetune(minibatch_index)
117 iter = epoch * n_train_batches + minibatch_index
118
119 printout_acc += cost_ij / float(printout_frequency * minibatch_size)
120 if (iter+1) % printout_frequency == 0:
121 print iter, "cost avg=", printout_acc
122 printout_acc = 0.0
123
124 if (iter+1) % validation_frequency == 0:
125
126 validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
127 this_validation_loss = numpy.mean(validation_losses)
128 print('epoch %i, minibatch %i/%i, validation error %f %%' % \
129 (epoch, minibatch_index+1, n_train_batches, \
130 this_validation_loss*100.))
131
132
133 # if we got the best validation score until now
134 if this_validation_loss < best_validation_loss:
135
136 #improve patience if loss improvement is good enough
137 if this_validation_loss < best_validation_loss * \
138 improvement_threshold :
139 patience = max(patience, iter * patience_increase)
140
141 # save best validation score and iteration number
142 best_validation_loss = this_validation_loss
143 best_iter = iter
144
145 # test it on the test set
146 test_losses = [test_model(i) for i in xrange(n_test_batches)]
147 test_score = numpy.mean(test_losses)
148 print((' epoch %i, minibatch %i/%i, test error of best '
149 'model %f %%') %
150 (epoch, minibatch_index+1, n_train_batches,
151 test_score*100.))
152
153
154 if patience <= iter :
155 done_looping = True
156 break
157
158 end_time = time.clock()
159 print(('Optimization complete with best validation score of %f %%,'
160 'with test performance %f %%') %
161
162 (best_validation_loss * 100., test_score*100.))
163 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
164
165