Mercurial > ift6266
annotate baseline/mlp/mlp_nist.py @ 404:1509b9bba4cc
added digit/char error
| author | xaviermuller |
|---|---|
| date | Wed, 28 Apr 2010 11:45:14 -0400 |
| parents | 60a4432b8071 |
| children | 195f95c3d461 |
| rev | line source |
|---|---|
| 110 | 1 """ |
| 2 This tutorial introduces the multilayer perceptron using Theano. | |
| 3 | |
| 4 A multilayer perceptron is a logistic regressor where | |
| 5 instead of feeding the input to the logistic regression you insert a | |
| 6 intermidiate layer, called the hidden layer, that has a nonlinear | |
| 7 activation function (usually tanh or sigmoid) . One can use many such | |
| 8 hidden layers making the architecture deep. The tutorial will also tackle | |
| 9 the problem of MNIST digit classification. | |
| 10 | |
| 11 .. math:: | |
| 12 | |
| 13 f(x) = G( b^{(2)} + W^{(2)}( s( b^{(1)} + W^{(1)} x))), | |
| 14 | |
| 15 References: | |
| 16 | |
| 17 - textbooks: "Pattern Recognition and Machine Learning" - | |
| 18 Christopher M. Bishop, section 5 | |
| 19 | |
| 20 TODO: recommended preprocessing, lr ranges, regularization ranges (explain | |
| 21 to do lr first, then add regularization) | |
| 22 | |
| 23 """ | |
| 24 __docformat__ = 'restructedtext en' | |
| 25 | |
|
355
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
26 import sys |
| 110 | 27 import pdb |
| 28 import numpy | |
| 29 import pylab | |
| 30 import theano | |
| 31 import theano.tensor as T | |
| 32 import time | |
| 33 import theano.tensor.nnet | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
34 import pylearn |
|
304
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
35 import theano,pylearn.version,ift6266 |
| 110 | 36 from pylearn.io import filetensor as ft |
| 322 | 37 from ift6266 import datasets |
| 110 | 38 |
| 39 data_path = '/data/lisa/data/nist/by_class/' | |
| 40 | |
| 41 class MLP(object): | |
| 42 """Multi-Layer Perceptron Class | |
| 43 | |
| 44 A multilayer perceptron is a feedforward artificial neural network model | |
| 45 that has one layer or more of hidden units and nonlinear activations. | |
| 46 Intermidiate layers usually have as activation function thanh or the | |
| 47 sigmoid function while the top layer is a softamx layer. | |
| 48 """ | |
| 49 | |
| 50 | |
| 51 | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
52 def __init__(self, input, n_in, n_hidden, n_out,learning_rate): |
| 110 | 53 """Initialize the parameters for the multilayer perceptron |
| 54 | |
| 55 :param input: symbolic variable that describes the input of the | |
| 56 architecture (one minibatch) | |
| 57 | |
| 58 :param n_in: number of input units, the dimension of the space in | |
| 59 which the datapoints lie | |
| 60 | |
| 61 :param n_hidden: number of hidden units | |
| 62 | |
| 63 :param n_out: number of output units, the dimension of the space in | |
| 64 which the labels lie | |
| 65 | |
| 66 """ | |
| 67 | |
| 68 # initialize the parameters theta = (W1,b1,W2,b2) ; note that this | |
| 69 # example contains only one hidden layer, but one can have as many | |
| 70 # layers as he/she wishes, making the network deeper. The only | |
| 71 # problem making the network deep this way is during learning, | |
| 72 # backpropagation being unable to move the network from the starting | |
| 73 # point towards; this is where pre-training helps, giving a good | |
| 74 # starting point for backpropagation, but more about this in the | |
| 75 # other tutorials | |
| 76 | |
| 77 # `W1` is initialized with `W1_values` which is uniformely sampled | |
| 78 # from -6./sqrt(n_in+n_hidden) and 6./sqrt(n_in+n_hidden) | |
| 79 # the output of uniform if converted using asarray to dtype | |
| 80 # theano.config.floatX so that the code is runable on GPU | |
| 81 W1_values = numpy.asarray( numpy.random.uniform( \ | |
| 82 low = -numpy.sqrt(6./(n_in+n_hidden)), \ | |
| 83 high = numpy.sqrt(6./(n_in+n_hidden)), \ | |
| 84 size = (n_in, n_hidden)), dtype = theano.config.floatX) | |
| 85 # `W2` is initialized with `W2_values` which is uniformely sampled | |
| 86 # from -6./sqrt(n_hidden+n_out) and 6./sqrt(n_hidden+n_out) | |
| 87 # the output of uniform if converted using asarray to dtype | |
| 88 # theano.config.floatX so that the code is runable on GPU | |
| 89 W2_values = numpy.asarray( numpy.random.uniform( | |
| 90 low = -numpy.sqrt(6./(n_hidden+n_out)), \ | |
| 91 high= numpy.sqrt(6./(n_hidden+n_out)),\ | |
| 92 size= (n_hidden, n_out)), dtype = theano.config.floatX) | |
| 93 | |
| 94 self.W1 = theano.shared( value = W1_values ) | |
| 95 self.b1 = theano.shared( value = numpy.zeros((n_hidden,), | |
| 96 dtype= theano.config.floatX)) | |
| 97 self.W2 = theano.shared( value = W2_values ) | |
| 98 self.b2 = theano.shared( value = numpy.zeros((n_out,), | |
| 99 dtype= theano.config.floatX)) | |
| 100 | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
101 #include the learning rate in the classifer so |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
102 #we can modify it on the fly when we want |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
103 lr_value=learning_rate |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
104 self.lr=theano.shared(value=lr_value) |
| 110 | 105 # symbolic expression computing the values of the hidden layer |
| 106 self.hidden = T.tanh(T.dot(input, self.W1)+ self.b1) | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
107 |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
108 |
| 110 | 109 |
| 110 # symbolic expression computing the values of the top layer | |
| 111 self.p_y_given_x= T.nnet.softmax(T.dot(self.hidden, self.W2)+self.b2) | |
| 112 | |
| 113 # compute prediction as class whose probability is maximal in | |
| 114 # symbolic form | |
| 115 self.y_pred = T.argmax( self.p_y_given_x, axis =1) | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
116 self.y_pred_num = T.argmax( self.p_y_given_x[0:9], axis =1) |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
117 |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
118 |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
119 |
| 110 | 120 |
| 121 # L1 norm ; one regularization option is to enforce L1 norm to | |
| 122 # be small | |
| 123 self.L1 = abs(self.W1).sum() + abs(self.W2).sum() | |
| 124 | |
| 125 # square of L2 norm ; one regularization option is to enforce | |
| 126 # square of L2 norm to be small | |
| 127 self.L2_sqr = (self.W1**2).sum() + (self.W2**2).sum() | |
| 128 | |
| 129 | |
| 130 | |
| 131 def negative_log_likelihood(self, y): | |
| 132 """Return the mean of the negative log-likelihood of the prediction | |
| 133 of this model under a given target distribution. | |
| 134 | |
| 135 .. math:: | |
| 136 | |
| 137 \frac{1}{|\mathcal{D}|}\mathcal{L} (\theta=\{W,b\}, \mathcal{D}) = | |
| 138 \frac{1}{|\mathcal{D}|}\sum_{i=0}^{|\mathcal{D}|} \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ | |
| 139 \ell (\theta=\{W,b\}, \mathcal{D}) | |
| 140 | |
| 141 | |
| 142 :param y: corresponds to a vector that gives for each example the | |
| 143 :correct label | |
| 144 """ | |
| 145 return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]),y]) | |
| 146 | |
| 147 | |
| 148 | |
| 149 | |
| 150 def errors(self, y): | |
| 151 """Return a float representing the number of errors in the minibatch | |
| 152 over the total number of examples of the minibatch | |
| 153 """ | |
| 154 | |
| 155 # check if y has same dimension of y_pred | |
| 156 if y.ndim != self.y_pred.ndim: | |
| 157 raise TypeError('y should have the same shape as self.y_pred', | |
| 158 ('y', target.type, 'y_pred', self.y_pred.type)) | |
| 159 # check if y is of the correct datatype | |
| 160 if y.dtype.startswith('int'): | |
| 161 # the T.neq operator returns a vector of 0s and 1s, where 1 | |
| 162 # represents a mistake in prediction | |
| 163 return T.mean(T.neq(self.y_pred, y)) | |
| 164 else: | |
| 165 raise NotImplementedError() | |
| 166 | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
167 def mlp_get_nist_error(model_name='/u/mullerx/ift6266h10_sandbox_db/xvm_final_lr1_p073/8/best_model.npy.npz', |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
168 data_set=0): |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
169 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
170 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
171 |
| 404 | 172 |
| 173 | |
| 174 | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
175 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
176 # load the data set and create an mlp based on the dimensions of the model |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
177 model=numpy.load(model_name) |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
178 W1=model['W1'] |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
179 W2=model['W2'] |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
180 b1=model['b1'] |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
181 b2=model['b2'] |
| 404 | 182 |
| 183 total_error_count=0.0 | |
| 184 total_exemple_count=0.0 | |
| 185 | |
| 186 nb_error_count=0.0 | |
| 187 nb_exemple_count=0.0 | |
| 188 | |
| 189 char_error_count=0.0 | |
| 190 char_exemple_count=0.0 | |
| 191 | |
| 192 min_error_count=0.0 | |
| 193 min_exemple_count=0.0 | |
| 194 | |
| 195 maj_error_count=0.0 | |
| 196 maj_exemple_count=0.0 | |
| 197 | |
| 198 | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
199 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
200 if data_set==0: |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
201 dataset=datasets.nist_all() |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
202 elif data_set==1: |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
203 dataset=datasets.nist_P07() |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
204 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
205 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
206 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
207 #get the test error |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
208 #use a batch size of 1 so we can get the sub-class error |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
209 #without messing with matrices (will be upgraded later) |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
210 test_score=0 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
211 temp=0 |
| 404 | 212 for xt,yt in dataset.test(1): |
| 213 | |
| 214 total_exemple_count = total_exemple_count +1 | |
| 215 #get activation for layer 1 | |
| 216 a0=numpy.dot(numpy.transpose(W1),numpy.transpose(xt[0])) + b1 | |
| 217 #add non linear function to layer 1 activation | |
| 218 a0_out=numpy.tanh(a0) | |
| 219 | |
| 220 #get activation for output layer | |
| 221 a1= numpy.dot(numpy.transpose(W2),a0_out) + b2 | |
| 222 #add non linear function for output activation (softmax) | |
| 223 a1_exp = numpy.exp(a1) | |
| 224 sum_a1=numpy.sum(a1_exp) | |
| 225 a1_out=a1_exp/sum_a1 | |
| 226 | |
| 227 predicted_class=numpy.argmax(a1_out) | |
| 228 wanted_class=yt[0] | |
| 229 if(predicted_class!=wanted_class): | |
| 230 total_error_count = total_error_count +1 | |
| 231 | |
| 232 #treat digit error | |
| 233 if(wanted_class<10): | |
| 234 nb_exemple_count=nb_exemple_count + 1 | |
| 235 predicted_class=numpy.argmax(a1_out[0:10]) | |
| 236 if(predicted_class!=wanted_class): | |
| 237 nb_error_count = nb_error_count +1 | |
| 238 | |
| 239 if(wanted_class>9): | |
| 240 char_exemple_count=char_exemple_count + 1 | |
| 241 predicted_class=numpy.argmax(a1_out[10:62])+10 | |
| 242 if((predicted_class!=wanted_class) and ((predicted_class+26)!=wanted_class) and ((predicted_class-26)!=wanted_class)): | |
| 243 char_error_count = char_error_count +1 | |
| 244 | |
| 245 | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
246 |
| 404 | 247 print (('total error = %f') % ((total_error_count/total_exemple_count)*100.0)) |
| 248 print (('number error = %f') % ((nb_error_count/nb_exemple_count)*100.0)) | |
| 249 print (('char error = %f') % ((char_error_count/char_exemple_count)*100.0)) | |
| 250 return (total_error_count/total_exemple_count)*100.0 | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
251 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
252 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
253 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
254 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
255 |
| 110 | 256 |
|
304
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
257 def mlp_full_nist( verbose = 1,\ |
|
145
8ceaaf812891
changed adaptive lr flag from bool to int for jobman issues
XavierMuller
parents:
143
diff
changeset
|
258 adaptive_lr = 0,\ |
| 322 | 259 data_set=0,\ |
| 110 | 260 learning_rate=0.01,\ |
| 261 L1_reg = 0.00,\ | |
| 262 L2_reg = 0.0001,\ | |
| 263 nb_max_exemples=1000000,\ | |
| 264 batch_size=20,\ | |
| 322 | 265 nb_hidden = 30,\ |
|
212
e390b0454515
added classic lr time decay and py code to calculate the error based on a saved model
xaviermuller
parents:
169
diff
changeset
|
266 nb_targets = 62, |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
267 tau=1e6,\ |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
268 lr_t2_factor=0.5,\ |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
269 init_model=0,\ |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
270 channel=0): |
| 110 | 271 |
| 272 | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
273 if channel!=0: |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
274 channel.save() |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
275 configuration = [learning_rate,nb_max_exemples,nb_hidden,adaptive_lr] |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
276 |
|
212
e390b0454515
added classic lr time decay and py code to calculate the error based on a saved model
xaviermuller
parents:
169
diff
changeset
|
277 #save initial learning rate if classical adaptive lr is used |
|
e390b0454515
added classic lr time decay and py code to calculate the error based on a saved model
xaviermuller
parents:
169
diff
changeset
|
278 initial_lr=learning_rate |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
279 max_div_count=1000 |
| 323 | 280 |
|
212
e390b0454515
added classic lr time decay and py code to calculate the error based on a saved model
xaviermuller
parents:
169
diff
changeset
|
281 |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
282 total_validation_error_list = [] |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
283 total_train_error_list = [] |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
284 learning_rate_list=[] |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
285 best_training_error=float('inf'); |
| 323 | 286 divergence_flag_list=[] |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
287 |
| 322 | 288 if data_set==0: |
| 378 | 289 print 'using nist' |
| 322 | 290 dataset=datasets.nist_all() |
| 323 | 291 elif data_set==1: |
| 378 | 292 print 'using p07' |
| 323 | 293 dataset=datasets.nist_P07() |
|
349
22efb4968054
added pnist support, will check in code for data set iterator later
xaviermuller
parents:
338
diff
changeset
|
294 elif data_set==2: |
| 378 | 295 print 'using pnist' |
|
349
22efb4968054
added pnist support, will check in code for data set iterator later
xaviermuller
parents:
338
diff
changeset
|
296 dataset=datasets.PNIST07() |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
297 |
|
212
e390b0454515
added classic lr time decay and py code to calculate the error based on a saved model
xaviermuller
parents:
169
diff
changeset
|
298 |
| 110 | 299 |
| 300 | |
| 301 ishape = (32,32) # this is the size of NIST images | |
| 302 | |
| 303 # allocate symbolic variables for the data | |
| 304 x = T.fmatrix() # the data is presented as rasterized images | |
| 305 y = T.lvector() # the labels are presented as 1D vector of | |
| 306 # [long int] labels | |
| 307 | |
| 322 | 308 |
| 110 | 309 # construct the logistic regression class |
| 322 | 310 classifier = MLP( input=x,\ |
| 110 | 311 n_in=32*32,\ |
| 312 n_hidden=nb_hidden,\ | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
313 n_out=nb_targets, |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
314 learning_rate=learning_rate) |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
315 |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
316 |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
317 # check if we want to initialise the weights with a previously calculated model |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
318 # dimensions must be consistent between old model and current configuration!!!!!! (nb_hidden and nb_targets) |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
319 if init_model!=0: |
| 378 | 320 print 'using old model' |
| 321 print init_model | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
322 old_model=numpy.load(init_model) |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
323 classifier.W1.value=old_model['W1'] |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
324 classifier.W2.value=old_model['W2'] |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
325 classifier.b1.value=old_model['b1'] |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
326 classifier.b2.value=old_model['b2'] |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
327 |
| 110 | 328 |
| 329 # the cost we minimize during training is the negative log likelihood of | |
| 330 # the model plus the regularization terms (L1 and L2); cost is expressed | |
| 331 # here symbolically | |
| 332 cost = classifier.negative_log_likelihood(y) \ | |
| 333 + L1_reg * classifier.L1 \ | |
| 334 + L2_reg * classifier.L2_sqr | |
| 335 | |
| 336 # compiling a theano function that computes the mistakes that are made by | |
| 337 # the model on a minibatch | |
| 338 test_model = theano.function([x,y], classifier.errors(y)) | |
| 339 | |
| 340 # compute the gradient of cost with respect to theta = (W1, b1, W2, b2) | |
| 341 g_W1 = T.grad(cost, classifier.W1) | |
| 342 g_b1 = T.grad(cost, classifier.b1) | |
| 343 g_W2 = T.grad(cost, classifier.W2) | |
| 344 g_b2 = T.grad(cost, classifier.b2) | |
| 345 | |
| 346 # specify how to update the parameters of the model as a dictionary | |
| 347 updates = \ | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
348 { classifier.W1: classifier.W1 - classifier.lr*g_W1 \ |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
349 , classifier.b1: classifier.b1 - classifier.lr*g_b1 \ |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
350 , classifier.W2: classifier.W2 - classifier.lr*g_W2 \ |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
351 , classifier.b2: classifier.b2 - classifier.lr*g_b2 } |
| 110 | 352 |
| 353 # compiling a theano function `train_model` that returns the cost, but in | |
| 354 # the same time updates the parameter of the model based on the rules | |
| 355 # defined in `updates` | |
| 356 train_model = theano.function([x, y], cost, updates = updates ) | |
| 322 | 357 |
| 358 | |
| 359 | |
| 110 | 360 |
| 361 | |
| 362 | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
363 |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
364 |
| 110 | 365 |
| 366 #conditions for stopping the adaptation: | |
| 323 | 367 #1) we have reached nb_max_exemples (this is rounded up to be a multiple of the train size so we always do at least 1 epoch) |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
368 #2) validation error is going up twice in a row(probable overfitting) |
| 110 | 369 |
| 370 # This means we no longer stop on slow convergence as low learning rates stopped | |
| 323 | 371 # too fast but instead we will wait for the valid error going up 3 times in a row |
| 372 # We save the curb of the validation error so we can always go back to check on it | |
| 373 # and we save the absolute best model anyway, so we might as well explore | |
| 374 # a bit when diverging | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
375 |
| 323 | 376 #approximate number of samples in the nist training set |
| 322 | 377 #this is just to have a validation frequency |
| 323 | 378 #roughly proportionnal to the original nist training set |
| 322 | 379 n_minibatches = 650000/batch_size |
| 380 | |
| 381 | |
| 323 | 382 patience =2*nb_max_exemples/batch_size #in units of minibatch |
| 110 | 383 validation_frequency = n_minibatches/4 |
| 384 | |
| 385 | |
| 386 | |
| 387 | |
| 322 | 388 |
| 110 | 389 best_validation_loss = float('inf') |
| 390 best_iter = 0 | |
| 391 test_score = 0. | |
| 392 start_time = time.clock() | |
|
212
e390b0454515
added classic lr time decay and py code to calculate the error based on a saved model
xaviermuller
parents:
169
diff
changeset
|
393 time_n=0 #in unit of exemples |
| 322 | 394 minibatch_index=0 |
| 395 epoch=0 | |
| 396 temp=0 | |
| 323 | 397 divergence_flag=0 |
| 322 | 398 |
|
212
e390b0454515
added classic lr time decay and py code to calculate the error based on a saved model
xaviermuller
parents:
169
diff
changeset
|
399 |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
400 |
|
355
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
401 |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
402 print 'starting training' |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
403 sys.stdout.flush() |
| 322 | 404 while(minibatch_index*batch_size<nb_max_exemples): |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
405 |
| 322 | 406 for x, y in dataset.train(batch_size): |
| 110 | 407 |
| 323 | 408 #if we are using the classic learning rate deacay, adjust it before training of current mini-batch |
| 322 | 409 if adaptive_lr==2: |
| 410 classifier.lr.value = tau*initial_lr/(tau+time_n) | |
| 411 | |
| 412 | |
| 413 #train model | |
| 414 cost_ij = train_model(x,y) | |
| 415 | |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
416 if (minibatch_index) % validation_frequency == 0: |
| 322 | 417 #save the current learning rate |
| 418 learning_rate_list.append(classifier.lr.value) | |
| 323 | 419 divergence_flag_list.append(divergence_flag) |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
420 |
|
355
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
421 |
| 322 | 422 |
| 423 # compute the validation error | |
| 424 this_validation_loss = 0. | |
| 425 temp=0 | |
| 426 for xv,yv in dataset.valid(1): | |
| 427 # sum up the errors for each minibatch | |
| 323 | 428 this_validation_loss += test_model(xv,yv) |
| 322 | 429 temp=temp+1 |
| 430 # get the average by dividing with the number of minibatches | |
| 431 this_validation_loss /= temp | |
| 432 #save the validation loss | |
| 433 total_validation_error_list.append(this_validation_loss) | |
|
355
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
434 |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
435 print(('epoch %i, minibatch %i, learning rate %f current validation error %f ') % |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
436 (epoch, minibatch_index+1,classifier.lr.value, |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
437 this_validation_loss*100.)) |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
438 sys.stdout.flush() |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
439 |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
440 #save temp results to check during training |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
441 numpy.savez('temp_results.npy',config=configuration,total_validation_error_list=total_validation_error_list,\ |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
442 learning_rate_list=learning_rate_list, divergence_flag_list=divergence_flag_list) |
| 322 | 443 |
| 444 # if we got the best validation score until now | |
| 445 if this_validation_loss < best_validation_loss: | |
| 446 # save best validation score and iteration number | |
| 447 best_validation_loss = this_validation_loss | |
| 448 best_iter = minibatch_index | |
| 323 | 449 #reset divergence flag |
| 450 divergence_flag=0 | |
| 451 | |
| 452 #save the best model. Overwrite the current saved best model so | |
| 453 #we only keep the best | |
| 454 numpy.savez('best_model.npy', config=configuration, W1=classifier.W1.value, W2=classifier.W2.value, b1=classifier.b1.value,\ | |
| 455 b2=classifier.b2.value, minibatch_index=minibatch_index) | |
| 456 | |
| 322 | 457 # test it on the test set |
| 458 test_score = 0. | |
| 459 temp =0 | |
| 460 for xt,yt in dataset.test(batch_size): | |
| 461 test_score += test_model(xt,yt) | |
| 462 temp = temp+1 | |
| 463 test_score /= temp | |
|
355
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
464 |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
465 print(('epoch %i, minibatch %i, test error of best ' |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
466 'model %f %%') % |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
467 (epoch, minibatch_index+1, |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
468 test_score*100.)) |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
469 sys.stdout.flush() |
| 322 | 470 |
| 471 # if the validation error is going up, we are overfitting (or oscillating) | |
| 323 | 472 # check if we are allowed to continue and if we will adjust the learning rate |
| 322 | 473 elif this_validation_loss >= best_validation_loss: |
| 323 | 474 |
| 475 | |
| 476 # In non-classic learning rate decay, we modify the weight only when | |
| 477 # validation error is going up | |
| 478 if adaptive_lr==1: | |
| 479 classifier.lr.value=classifier.lr.value*lr_t2_factor | |
| 480 | |
| 481 | |
| 482 #cap the patience so we are allowed to diverge max_div_count times | |
| 483 #if we are going up max_div_count in a row, we will stop immediatelty by modifying the patience | |
| 484 divergence_flag = divergence_flag +1 | |
| 485 | |
| 486 | |
| 322 | 487 #calculate the test error at this point and exit |
| 488 # test it on the test set | |
| 489 test_score = 0. | |
| 490 temp=0 | |
| 491 for xt,yt in dataset.test(batch_size): | |
| 492 test_score += test_model(xt,yt) | |
| 493 temp=temp+1 | |
| 494 test_score /= temp | |
|
355
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
495 |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
496 print ' validation error is going up, possibly stopping soon' |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
497 print((' epoch %i, minibatch %i, test error of best ' |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
498 'model %f %%') % |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
499 (epoch, minibatch_index+1, |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
500 test_score*100.)) |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
501 sys.stdout.flush() |
| 322 | 502 |
| 503 | |
| 504 | |
| 323 | 505 # check early stop condition |
| 506 if divergence_flag==max_div_count: | |
| 507 minibatch_index=nb_max_exemples | |
| 508 print 'we have diverged, early stopping kicks in' | |
| 509 break | |
| 510 | |
| 511 #check if we have seen enough exemples | |
| 512 #force one epoch at least | |
| 513 if epoch>0 and minibatch_index*batch_size>nb_max_exemples: | |
| 322 | 514 break |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
515 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
516 |
|
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
517 |
| 322 | 518 |
| 519 | |
| 520 time_n= time_n + batch_size | |
| 323 | 521 minibatch_index = minibatch_index + 1 |
| 522 | |
| 523 # we have finished looping through the training set | |
| 322 | 524 epoch = epoch+1 |
| 110 | 525 end_time = time.clock() |
|
355
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
526 |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
527 print(('Optimization complete. Best validation score of %f %% ' |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
528 'obtained at iteration %i, with test performance %f %%') % |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
529 (best_validation_loss * 100., best_iter, test_score*100.)) |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
530 print ('The code ran for %f minutes' % ((end_time-start_time)/60.)) |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
531 print minibatch_index |
|
76b7182dd32e
added support for pnist in iterator. corrected a print bug in mlp
xaviermuller
parents:
349
diff
changeset
|
532 sys.stdout.flush() |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
533 |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
534 #save the model and the weights |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
535 numpy.savez('model.npy', config=configuration, W1=classifier.W1.value,W2=classifier.W2.value, b1=classifier.b1.value,b2=classifier.b2.value) |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
536 numpy.savez('results.npy',config=configuration,total_train_error_list=total_train_error_list,total_validation_error_list=total_validation_error_list,\ |
| 323 | 537 learning_rate_list=learning_rate_list, divergence_flag_list=divergence_flag_list) |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
538 |
|
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
539 return (best_training_error*100.0,best_validation_loss * 100.,test_score*100.,best_iter*batch_size,(end_time-start_time)/60) |
| 110 | 540 |
| 541 | |
| 542 if __name__ == '__main__': | |
| 543 mlp_full_mnist() | |
| 544 | |
| 545 def jobman_mlp_full_nist(state,channel): | |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
546 (train_error,validation_error,test_error,nb_exemples,time)=mlp_full_nist(learning_rate=state.learning_rate,\ |
|
304
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
547 nb_max_exemples=state.nb_max_exemples,\ |
|
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
548 nb_hidden=state.nb_hidden,\ |
|
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
549 adaptive_lr=state.adaptive_lr,\ |
|
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
550 tau=state.tau,\ |
|
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
551 verbose = state.verbose,\ |
| 324 | 552 lr_t2_factor=state.lr_t2_factor, |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
553 data_set=state.data_set, |
| 378 | 554 init_model=state.init_model, |
|
338
fca22114bb23
added async save, restart from old model and independant error calculation based on Arnaud's iterator
xaviermuller
parents:
324
diff
changeset
|
555 channel=channel) |
|
143
f341a4efb44a
added adaptive lr, weight file save, traine error and error curves
XavierMuller
parents:
110
diff
changeset
|
556 state.train_error=train_error |
| 110 | 557 state.validation_error=validation_error |
| 558 state.test_error=test_error | |
| 559 state.nb_exemples=nb_exemples | |
| 560 state.time=time | |
|
304
1e4bf5a5b46d
added type 2 adaptive learning configurable learning weight + versionning
xaviermuller
parents:
237
diff
changeset
|
561 pylearn.version.record_versions(state,[theano,ift6266,pylearn]) |
| 110 | 562 return channel.COMPLETE |
| 563 | |
| 564 |