ift6266: baseline/mlp/mlp_nist.py comparison

comparison baseline/mlp/mlp_nist.py @ 322:743907366476

code clean up in progress

author	xaviermuller
date	Tue, 06 Apr 2010 16:00:52 -0400
parents	1e4bf5a5b46d
children	7a7615f940e8

comparison

equal deleted inserted replaced

-:403b9e6ecfaa
+:743907366476
 import time
 import theano.tensor.nnet
 import pylearn
 import theano,pylearn.version,ift6266
 from pylearn.io import filetensor as ft
+from ift6266 import datasets
 data_path = '/data/lisa/data/nist/by_class/'
 class MLP(object):
 """Multi-Layer Perceptron Class
 raise NotImplementedError()
 def mlp_full_nist(      verbose = 1,\
 adaptive_lr = 0,\
-train_data = 'all/all_train_data.ft',\
+data_set=0,\
-train_labels = 'all/all_train_labels.ft',\
-test_data = 'all/all_test_data.ft',\
-test_labels = 'all/all_test_labels.ft',\
 learning_rate=0.01,\
 L1_reg = 0.00,\
 L2_reg = 0.0001,\
 nb_max_exemples=1000000,\
 batch_size=20,\
-nb_hidden = 500,\
+nb_hidden = 30,\
 nb_targets = 62,
 			tau=1e6,\
 			lr_t2_factor=0.5):
 total_validation_error_list = []
 total_train_error_list = []
 learning_rate_list=[]
 best_training_error=float('inf');
+if data_set==0:
+	dataset=datasets.nist_all()
-f = open(data_path+train_data)
-g= open(data_path+train_labels)
-h = open(data_path+test_data)
-i= open(data_path+test_labels)
-raw_train_data = ft.read(f)
-raw_train_labels = ft.read(g)
-raw_test_data = ft.read(h)
-raw_test_labels = ft.read(i)
-f.close()
-g.close()
-i.close()
-h.close()
-#create a validation set the same size as the test size
-#use the end of the training array for this purpose
-#discard the last remaining so we get a %batch_size number
-test_size=len(raw_test_labels)
-test_size = int(test_size/batch_size)
-test_size*=batch_size
-train_size = len(raw_train_data)
-train_size = int(train_size/batch_size)
-train_size*=batch_size
-validation_size =test_size
-offset = train_size-test_size
-if verbose == 1:
-print 'train size = %d' %train_size
-print 'test size = %d' %test_size
-print 'valid size = %d' %validation_size
-print 'offset = %d' %offset
-train_set = (raw_train_data,raw_train_labels)
-train_batches = []
-for i in xrange(0, train_size-test_size, batch_size):
-train_batches = train_batches + \
-[(raw_train_data[i:i+batch_size], raw_train_labels[i:i+batch_size])]
-test_batches = []
-for i in xrange(0, test_size, batch_size):
-test_batches = test_batches + \
-[(raw_test_data[i:i+batch_size], raw_test_labels[i:i+batch_size])]
-validation_batches = []
-for i in xrange(0, test_size, batch_size):
-validation_batches = validation_batches + \
-[(raw_train_data[offset+i:offset+i+batch_size], raw_train_labels[offset+i:offset+i+batch_size])]
 ishape     = (32,32) # this is the size of NIST images
 # allocate symbolic variables for the data
 x = T.fmatrix()  # the data is presented as rasterized images
 y = T.lvector()  # the labels are presented as 1D vector of
 # [long int] labels
-if verbose==1:
-print 'finished parsing the data'
 # construct the logistic regression class
-classifier = MLP( input=x.reshape((batch_size,32*32)),\
+classifier = MLP( input=x,\
 n_in=32*32,\
 n_hidden=nb_hidden,\
 n_out=nb_targets,
 learning_rate=learning_rate)
 # compiling a theano function `train_model` that returns the cost, but in
 # the same time updates the parameter of the model based on the rules
 # defined in `updates`
 train_model = theano.function([x, y], cost, updates = updates )
-n_minibatches        = len(train_batches)
 #2) validation error is going up twice in a row(probable overfitting)
 # This means we no longer stop on slow convergence as low learning rates stopped
 # too fast.
-# no longer relevant
+#approximate number of samples in the training set
-patience              =nb_max_exemples/batch_size
+#this is just to have a validation frequency
+#roughly proportionnal to the training set
+n_minibatches        = 650000/batch_size
+patience              =nb_max_exemples/batch_size #in units of minibatch
 patience_increase     = 2     # wait this much longer when a new best is
 # found
 improvement_threshold = 0.995 # a relative improvement of this much is
 # considered significant
 validation_frequency = n_minibatches/4
-best_params          = None
 best_validation_loss = float('inf')
 best_iter            = 0
 test_score           = 0.
 start_time = time.clock()
-n_iter = nb_max_exemples/batch_size  # nb of max times we are allowed to run through all exemples
-n_iter = n_iter/n_minibatches + 1 #round up
-n_iter=max(1,n_iter) # run at least once on short debug call
 time_n=0 #in unit of exemples
+minibatch_index=0
+epoch=0
+temp=0
 if verbose == 1:
-print 'looping at most %d times through the data set' %n_iter
+print 'looking at most at %i exemples' %nb_max_exemples
-for iter in xrange(n_iter* n_minibatches):
+while(minibatch_index*batch_size<nb_max_exemples):
-# get epoch and minibatch index
+for x, y in dataset.train(batch_size):
-epoch           = iter / n_minibatches
-minibatch_index =  iter % n_minibatches
-	if adaptive_lr==2:
-	    classifier.lr.value = tau*initial_lr/(tau+time_n)
-# get the minibatches corresponding to `iter` modulo
-# `len(train_batches)`
-x,y = train_batches[ minibatch_index ]
-# convert to float
-x_float = x/255.0
-cost_ij = train_model(x_float,y)
-if (iter+1) % validation_frequency == 0:
-# compute zero-one loss on validation set
-this_validation_loss = 0.
+minibatch_index =  minibatch_index + 1
-for x,y in validation_batches:
+if adaptive_lr==2:
-# sum up the errors for each minibatch
+classifier.lr.value = tau*initial_lr/(tau+time_n)
-x_float = x/255.0
-this_validation_loss += test_model(x_float,y)
-# get the average by dividing with the number of minibatches
-this_validation_loss /= len(validation_batches)
-#save the validation loss
-total_validation_error_list.append(this_validation_loss)
-#get the training error rate
+#train model
-this_train_loss=0
+cost_ij = train_model(x,y)
-for x,y in train_batches:
-# sum up the errors for each minibatch
+if (minibatch_index+1) % validation_frequency == 0:
-x_float = x/255.0
-this_train_loss += test_model(x_float,y)
-# get the average by dividing with the number of minibatches
-this_train_loss /= len(train_batches)
-#save the validation loss
-total_train_error_list.append(this_train_loss)
-if(this_train_loss<best_training_error):
-best_training_error=this_train_loss
-if verbose == 1:
+#save the current learning rate
-print('epoch %i, minibatch %i/%i, validation error %f, training error %f %%' % \
+learning_rate_list.append(classifier.lr.value)
-(epoch, minibatch_index+1, n_minibatches, \
-this_validation_loss*100.,this_train_loss*100))
+# compute the validation error
-		print 'learning rate = %f' %classifier.lr.value
+this_validation_loss = 0.
-		print 'time  = %i' %time_n
+temp=0
+for xv,yv in dataset.valid(1):
+# sum up the errors for each minibatch
-#save the learning rate
+axxa=test_model(xv,yv)
-learning_rate_list.append(classifier.lr.value)
+this_validation_loss += axxa
+temp=temp+1
+# get the average by dividing with the number of minibatches
-# if we got the best validation score until now
+this_validation_loss /= temp
-if this_validation_loss < best_validation_loss:
+#save the validation loss
-# save best validation score and iteration number
+total_validation_error_list.append(this_validation_loss)
-best_validation_loss = this_validation_loss
-best_iter = iter
-# reset patience if we are going down again
-# so we continue exploring
-patience=nb_max_exemples/batch_size
-# test it on the test set
-test_score = 0.
-for x,y in test_batches:
-x_float=x/255.0
-test_score += test_model(x_float,y)
-test_score /= len(test_batches)
 if verbose == 1:
-print(('     epoch %i, minibatch %i/%i, test error of best '
+print(('epoch %i, minibatch %i, learning rate %f current validation error %f ') %
-'model %f %%') %
+(epoch, minibatch_index+1,classifier.lr.value,
-(epoch, minibatch_index+1, n_minibatches,
+this_validation_loss*100.))
-test_score*100.))
+# if we got the best validation score until now
-# if the validation error is going up, we are overfitting (or oscillating)
+if this_validation_loss < best_validation_loss:
-# stop converging but run at least to next validation
+# save best validation score and iteration number
-# to check overfitting or ocsillation
+best_validation_loss = this_validation_loss
-# the saved weights of the model will be a bit off in that case
+best_iter = minibatch_index
-elif this_validation_loss >= best_validation_loss:
+# reset patience if we are going down again
-#calculate the test error at this point and exit
+# so we continue exploring
-# test it on the test set
+patience=nb_max_exemples/batch_size
-# however, if adaptive_lr is true, try reducing the lr to
+# test it on the test set
-# get us out of an oscilliation
+test_score = 0.
-if adaptive_lr==1:
+temp =0
-classifier.lr.value=classifier.lr.value*lr_t2_factor
+for xt,yt in dataset.test(batch_size):
+test_score += test_model(xt,yt)
-test_score = 0.
+temp = temp+1
-#cap the patience so we are allowed one more validation error
+test_score /= temp
-#calculation before aborting
+if verbose == 1:
-patience = iter+validation_frequency+1
+print(('epoch %i, minibatch %i, test error of best '
-for x,y in test_batches:
+'model %f %%') %
-x_float=x/255.0
+(epoch, minibatch_index+1,
-test_score += test_model(x_float,y)
+test_score*100.))
-test_score /= len(test_batches)
-if verbose == 1:
+# if the validation error is going up, we are overfitting (or oscillating)
-print ' validation error is going up, possibly stopping soon'
+# stop converging but run at least to next validation
-print(('     epoch %i, minibatch %i/%i, test error of best '
+# to check overfitting or ocsillation
-'model %f %%') %
+# the saved weights of the model will be a bit off in that case
-(epoch, minibatch_index+1, n_minibatches,
+elif this_validation_loss >= best_validation_loss:
-test_score*100.))
+#calculate the test error at this point and exit
+# test it on the test set
+# however, if adaptive_lr is true, try reducing the lr to
+# get us out of an oscilliation
+if adaptive_lr==1:
-if iter>patience:
+classifier.lr.value=classifier.lr.value*lr_t2_factor
-print 'we have diverged'
-break
+test_score = 0.
+#cap the patience so we are allowed one more validation error
+#calculation before aborting
-	time_n= time_n + batch_size
+patience = minibatch_index+validation_frequency+1
+temp=0
+for xt,yt in dataset.test(batch_size):
+test_score += test_model(xt,yt)
+temp=temp+1
+test_score /= temp
+if verbose == 1:
+print ' validation error is going up, possibly stopping soon'
+print(('     epoch %i, minibatch %i, test error of best '
+'model %f %%') %
+(epoch, minibatch_index+1,
+test_score*100.))
+if minibatch_index>patience:
+print 'we have diverged'
+break
+time_n= time_n + batch_size
+epoch = epoch+1
 end_time = time.clock()
 if verbose == 1:
 print(('Optimization complete. Best validation score of %f %% '
 'obtained at iteration %i, with test performance %f %%') %
 (best_validation_loss * 100., best_iter, test_score*100.))
 print ('The code ran for %f minutes' % ((end_time-start_time)/60.))
-print iter
+print minibatch_index
 #save the model and the weights
 numpy.savez('model.npy', config=configuration, W1=classifier.W1.value,W2=classifier.W2.value, b1=classifier.b1.value,b2=classifier.b2.value)
 numpy.savez('results.npy',config=configuration,total_train_error_list=total_train_error_list,total_validation_error_list=total_validation_error_list,\
 learning_rate_list=learning_rate_list)

Mercurial > ift6266

comparison baseline/mlp/mlp_nist.py @ 322:743907366476