Mercurial > ift6266

comparison baseline/mlp/mlp_nist.py @ 304:1e4bf5a5b46d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added type 2 adaptive learning configurable learning weight + versionning
author xaviermuller
date Wed, 31 Mar 2010 16:06:55 -0400
parents 9b6e0af062af
children 743907366476
comparison
equal deleted inserted replaced
303:ef28cbb5f464 304:1e4bf5a5b46d
29 import theano 29 import theano
30 import theano.tensor as T 30 import theano.tensor as T
31 import time 31 import time
32 import theano.tensor.nnet 32 import theano.tensor.nnet
33 import pylearn 33 import pylearn
34 import theano,pylearn.version 34 import theano,pylearn.version,ift6266
35 from pylearn.io import filetensor as ft 35 from pylearn.io import filetensor as ft
36 36
37 data_path = '/data/lisa/data/nist/by_class/' 37 data_path = '/data/lisa/data/nist/by_class/'
38 38
39 class MLP(object): 39 class MLP(object):
161 return T.mean(T.neq(self.y_pred, y)) 161 return T.mean(T.neq(self.y_pred, y))
162 else: 162 else:
163 raise NotImplementedError() 163 raise NotImplementedError()
164 164
165 165
166 def mlp_full_nist( verbose = False,\ 166 def mlp_full_nist( verbose = 1,\
167 adaptive_lr = 0,\ 167 adaptive_lr = 0,\
168 train_data = 'all/all_train_data.ft',\ 168 train_data = 'all/all_train_data.ft',\
169 train_labels = 'all/all_train_labels.ft',\ 169 train_labels = 'all/all_train_labels.ft',\
170 test_data = 'all/all_test_data.ft',\ 170 test_data = 'all/all_test_data.ft',\
171 test_labels = 'all/all_test_labels.ft',\ 171 test_labels = 'all/all_test_labels.ft',\
174 L2_reg = 0.0001,\ 174 L2_reg = 0.0001,\
175 nb_max_exemples=1000000,\ 175 nb_max_exemples=1000000,\
176 batch_size=20,\ 176 batch_size=20,\
177 nb_hidden = 500,\ 177 nb_hidden = 500,\
178 nb_targets = 62, 178 nb_targets = 62,
179 tau=1e6): 179 tau=1e6,\
180 lr_t2_factor=0.5):
180 181
181 182
182 configuration = [learning_rate,nb_max_exemples,nb_hidden,adaptive_lr] 183 configuration = [learning_rate,nb_max_exemples,nb_hidden,adaptive_lr]
183 184
184 #save initial learning rate if classical adaptive lr is used 185 #save initial learning rate if classical adaptive lr is used
215 train_size = len(raw_train_data) 216 train_size = len(raw_train_data)
216 train_size = int(train_size/batch_size) 217 train_size = int(train_size/batch_size)
217 train_size*=batch_size 218 train_size*=batch_size
218 validation_size =test_size 219 validation_size =test_size
219 offset = train_size-test_size 220 offset = train_size-test_size
220 if verbose == True: 221 if verbose == 1:
221 print 'train size = %d' %train_size 222 print 'train size = %d' %train_size
222 print 'test size = %d' %test_size 223 print 'test size = %d' %test_size
223 print 'valid size = %d' %validation_size 224 print 'valid size = %d' %validation_size
224 print 'offset = %d' %offset 225 print 'offset = %d' %offset
225 226
246 # allocate symbolic variables for the data 247 # allocate symbolic variables for the data
247 x = T.fmatrix() # the data is presented as rasterized images 248 x = T.fmatrix() # the data is presented as rasterized images
248 y = T.lvector() # the labels are presented as 1D vector of 249 y = T.lvector() # the labels are presented as 1D vector of
249 # [long int] labels 250 # [long int] labels
250 251
251 if verbose==True: 252 if verbose==1:
252 print 'finished parsing the data' 253 print 'finished parsing the data'
253 # construct the logistic regression class 254 # construct the logistic regression class
254 classifier = MLP( input=x.reshape((batch_size,32*32)),\ 255 classifier = MLP( input=x.reshape((batch_size,32*32)),\
255 n_in=32*32,\ 256 n_in=32*32,\
256 n_hidden=nb_hidden,\ 257 n_hidden=nb_hidden,\
323 n_iter=max(1,n_iter) # run at least once on short debug call 324 n_iter=max(1,n_iter) # run at least once on short debug call
324 time_n=0 #in unit of exemples 325 time_n=0 #in unit of exemples
325 326
326 327
327 328
328 if verbose == True: 329 if verbose == 1:
329 print 'looping at most %d times through the data set' %n_iter 330 print 'looping at most %d times through the data set' %n_iter
330 for iter in xrange(n_iter* n_minibatches): 331 for iter in xrange(n_iter* n_minibatches):
331 332
332 # get epoch and minibatch index 333 # get epoch and minibatch index
333 epoch = iter / n_minibatches 334 epoch = iter / n_minibatches
369 #save the validation loss 370 #save the validation loss
370 total_train_error_list.append(this_train_loss) 371 total_train_error_list.append(this_train_loss)
371 if(this_train_loss<best_training_error): 372 if(this_train_loss<best_training_error):
372 best_training_error=this_train_loss 373 best_training_error=this_train_loss
373 374
374 if verbose == True: 375 if verbose == 1:
375 print('epoch %i, minibatch %i/%i, validation error %f, training error %f %%' % \ 376 print('epoch %i, minibatch %i/%i, validation error %f, training error %f %%' % \
376 (epoch, minibatch_index+1, n_minibatches, \ 377 (epoch, minibatch_index+1, n_minibatches, \
377 this_validation_loss*100.,this_train_loss*100)) 378 this_validation_loss*100.,this_train_loss*100))
378 print 'learning rate = %f' %classifier.lr.value 379 print 'learning rate = %f' %classifier.lr.value
379 print 'time = %i' %time_n 380 print 'time = %i' %time_n
395 test_score = 0. 396 test_score = 0.
396 for x,y in test_batches: 397 for x,y in test_batches:
397 x_float=x/255.0 398 x_float=x/255.0
398 test_score += test_model(x_float,y) 399 test_score += test_model(x_float,y)
399 test_score /= len(test_batches) 400 test_score /= len(test_batches)
400 if verbose == True: 401 if verbose == 1:
401 print((' epoch %i, minibatch %i/%i, test error of best ' 402 print((' epoch %i, minibatch %i/%i, test error of best '
402 'model %f %%') % 403 'model %f %%') %
403 (epoch, minibatch_index+1, n_minibatches, 404 (epoch, minibatch_index+1, n_minibatches,
404 test_score*100.)) 405 test_score*100.))
405 406
411 #calculate the test error at this point and exit 412 #calculate the test error at this point and exit
412 # test it on the test set 413 # test it on the test set
413 # however, if adaptive_lr is true, try reducing the lr to 414 # however, if adaptive_lr is true, try reducing the lr to
414 # get us out of an oscilliation 415 # get us out of an oscilliation
415 if adaptive_lr==1: 416 if adaptive_lr==1:
416 classifier.lr.value=classifier.lr.value/2.0 417 classifier.lr.value=classifier.lr.value*lr_t2_factor
417 418
418 test_score = 0. 419 test_score = 0.
419 #cap the patience so we are allowed one more validation error 420 #cap the patience so we are allowed one more validation error
420 #calculation before aborting 421 #calculation before aborting
421 patience = iter+validation_frequency+1 422 patience = iter+validation_frequency+1
422 for x,y in test_batches: 423 for x,y in test_batches:
423 x_float=x/255.0 424 x_float=x/255.0
424 test_score += test_model(x_float,y) 425 test_score += test_model(x_float,y)
425 test_score /= len(test_batches) 426 test_score /= len(test_batches)
426 if verbose == True: 427 if verbose == 1:
427 print ' validation error is going up, possibly stopping soon' 428 print ' validation error is going up, possibly stopping soon'
428 print((' epoch %i, minibatch %i/%i, test error of best ' 429 print((' epoch %i, minibatch %i/%i, test error of best '
429 'model %f %%') % 430 'model %f %%') %
430 (epoch, minibatch_index+1, n_minibatches, 431 (epoch, minibatch_index+1, n_minibatches,
431 test_score*100.)) 432 test_score*100.))
438 break 439 break
439 440
440 441
441 time_n= time_n + batch_size 442 time_n= time_n + batch_size
442 end_time = time.clock() 443 end_time = time.clock()
443 if verbose == True: 444 if verbose == 1:
444 print(('Optimization complete. Best validation score of %f %% ' 445 print(('Optimization complete. Best validation score of %f %% '
445 'obtained at iteration %i, with test performance %f %%') % 446 'obtained at iteration %i, with test performance %f %%') %
446 (best_validation_loss * 100., best_iter, test_score*100.)) 447 (best_validation_loss * 100., best_iter, test_score*100.))
447 print ('The code ran for %f minutes' % ((end_time-start_time)/60.)) 448 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
448 print iter 449 print iter
458 if __name__ == '__main__': 459 if __name__ == '__main__':
459 mlp_full_mnist() 460 mlp_full_mnist()
460 461
461 def jobman_mlp_full_nist(state,channel): 462 def jobman_mlp_full_nist(state,channel):
462 (train_error,validation_error,test_error,nb_exemples,time)=mlp_full_nist(learning_rate=state.learning_rate,\ 463 (train_error,validation_error,test_error,nb_exemples,time)=mlp_full_nist(learning_rate=state.learning_rate,\
463 nb_max_exemples=state.nb_max_exemples,\ 464 nb_max_exemples=state.nb_max_exemples,\
464 nb_hidden=state.nb_hidden,\ 465 nb_hidden=state.nb_hidden,\
465 adaptive_lr=state.adaptive_lr,\ 466 adaptive_lr=state.adaptive_lr,\
466 tau=state.tau) 467 tau=state.tau,\
468 verbose = state.verbose,\
469 train_data = state.train_data,\
470 train_labels = state.train_labels,\
471 test_data = state.test_data,\
472 test_labels = state.test_labels,\
473 lr_t2_factor=state.lr_t2_factor)
467 state.train_error=train_error 474 state.train_error=train_error
468 state.validation_error=validation_error 475 state.validation_error=validation_error
469 state.test_error=test_error 476 state.test_error=test_error
470 state.nb_exemples=nb_exemples 477 state.nb_exemples=nb_exemples
471 state.time=time 478 state.time=time
479 pylearn.version.record_versions(state,[theano,ift6266,pylearn])
472 return channel.COMPLETE 480 return channel.COMPLETE
473 481
474 482