# HG changeset patch # User XavierMuller # Date 1266967015 18000 # Node ID c958941c1b9de8e7583982d66a7491bc58c9b0b1 # Parent f341a4efb44a57634c7428823f4c853819998ec3# Parent bb26c12bb9f61b4314bf952c25c54ab6290a45d0 merge diff -r bb26c12bb9f6 -r c958941c1b9d baseline_algorithms/mlp/mlp_nist.py --- a/baseline_algorithms/mlp/mlp_nist.py Tue Feb 23 14:15:47 2010 -0500 +++ b/baseline_algorithms/mlp/mlp_nist.py Tue Feb 23 18:16:55 2010 -0500 @@ -30,6 +30,7 @@ import theano.tensor as T import time import theano.tensor.nnet +import pylearn from pylearn.io import filetensor as ft data_path = '/data/lisa/data/nist/by_class/' @@ -45,7 +46,7 @@ - def __init__(self, input, n_in, n_hidden, n_out): + def __init__(self, input, n_in, n_hidden, n_out,learning_rate): """Initialize the parameters for the multilayer perceptron :param input: symbolic variable that describes the input of the @@ -94,8 +95,14 @@ self.b2 = theano.shared( value = numpy.zeros((n_out,), dtype= theano.config.floatX)) + #include the learning rate in the classifer so + #we can modify it on the fly when we want + lr_value=learning_rate + self.lr=theano.shared(value=lr_value) # symbolic expression computing the values of the hidden layer self.hidden = T.tanh(T.dot(input, self.W1)+ self.b1) + + # symbolic expression computing the values of the top layer self.p_y_given_x= T.nnet.softmax(T.dot(self.hidden, self.W2)+self.b2) @@ -103,6 +110,10 @@ # compute prediction as class whose probability is maximal in # symbolic form self.y_pred = T.argmax( self.p_y_given_x, axis =1) + self.y_pred_num = T.argmax( self.p_y_given_x[0:9], axis =1) + + + # L1 norm ; one regularization option is to enforce L1 norm to # be small @@ -150,21 +161,9 @@ else: raise NotImplementedError() -#def jobman_mlp(state,channel): -# (validation_error,test_error,nb_exemples,time)=mlp_full_nist(state.learning_rate,\ - # state.n_iter,\ - # state.batch_size,\ - # state.nb_hidden_units) - # state.validation_error = validation_error - # state.test_error = test_error - # state.nb_exemples = nb_exemples - # state.time=time - # return channel.COMPLETE - - - def mlp_full_nist( verbose = False,\ + adaptive_lr = False,\ train_data = 'all/all_train_data.ft',\ train_labels = 'all/all_train_labels.ft',\ test_data = 'all/all_test_data.ft',\ @@ -178,6 +177,14 @@ nb_targets = 62): + configuration = [learning_rate,nb_max_exemples,nb_hidden,adaptive_lr] + + total_validation_error_list = [] + total_train_error_list = [] + learning_rate_list=[] + best_training_error=float('inf'); + + f = open(data_path+train_data) g= open(data_path+train_labels) @@ -235,11 +242,17 @@ y = T.lvector() # the labels are presented as 1D vector of # [long int] labels + if verbose==True: + print 'finished parsing the data' # construct the logistic regression class classifier = MLP( input=x.reshape((batch_size,32*32)),\ n_in=32*32,\ n_hidden=nb_hidden,\ - n_out=nb_targets) + n_out=nb_targets, + learning_rate=learning_rate) + + + # the cost we minimize during training is the negative log likelihood of # the model plus the regularization terms (L1 and L2); cost is expressed @@ -260,10 +273,10 @@ # specify how to update the parameters of the model as a dictionary updates = \ - { classifier.W1: classifier.W1 - learning_rate*g_W1 \ - , classifier.b1: classifier.b1 - learning_rate*g_b1 \ - , classifier.W2: classifier.W2 - learning_rate*g_W2 \ - , classifier.b2: classifier.b2 - learning_rate*g_b2 } + { classifier.W1: classifier.W1 - classifier.lr*g_W1 \ + , classifier.b1: classifier.b1 - classifier.lr*g_b1 \ + , classifier.W2: classifier.W2 - classifier.lr*g_W2 \ + , classifier.b2: classifier.b2 - classifier.lr*g_b2 } # compiling a theano function `train_model` that returns the cost, but in # the same time updates the parameter of the model based on the rules @@ -273,13 +286,17 @@ + + #conditions for stopping the adaptation: #1) we have reached nb_max_exemples (this is rounded up to be a multiple of the train size) - #2) validation error is going up (probable overfitting) + #2) validation error is going up twice in a row(probable overfitting) # This means we no longer stop on slow convergence as low learning rates stopped # too fast. + + # no longer relevant patience =nb_max_exemples/batch_size patience_increase = 2 # wait this much longer when a new best is # found @@ -296,9 +313,9 @@ test_score = 0. start_time = time.clock() n_iter = nb_max_exemples/batch_size # nb of max times we are allowed to run through all exemples - n_iter = n_iter/n_minibatches + 1 + n_iter = n_iter/n_minibatches + 1 #round up n_iter=max(1,n_iter) # run at least once on short debug call - # have a maximum of `n_iter` iterations through the entire dataset + if verbose == True: print 'looping at most %d times through the data set' %n_iter @@ -307,7 +324,9 @@ # get epoch and minibatch index epoch = iter / n_minibatches minibatch_index = iter % n_minibatches - + + + # get the minibatches corresponding to `iter` modulo # `len(train_batches)` x,y = train_batches[ minibatch_index ] @@ -317,7 +336,7 @@ if (iter+1) % validation_frequency == 0: # compute zero-one loss on validation set - + this_validation_loss = 0. for x,y in validation_batches: # sum up the errors for each minibatch @@ -325,26 +344,40 @@ this_validation_loss += test_model(x_float,y) # get the average by dividing with the number of minibatches this_validation_loss /= len(validation_batches) + #save the validation loss + total_validation_error_list.append(this_validation_loss) + + #get the training error rate + this_train_loss=0 + for x,y in train_batches: + # sum up the errors for each minibatch + x_float = x/255.0 + this_train_loss += test_model(x_float,y) + # get the average by dividing with the number of minibatches + this_train_loss /= len(train_batches) + #save the validation loss + total_train_error_list.append(this_train_loss) + if(this_train_loss best_validation_loss: + # if the validation error is going up, we are overfitting (or oscillating) + # stop converging but run at least to next validation + # to check overfitting or ocsillation + # the saved weights of the model will be a bit off in that case + elif this_validation_loss >= best_validation_loss: #calculate the test error at this point and exit # test it on the test set - if verbose==True: - print ' We are diverging' - best_iter = iter + # however, if adaptive_lr is true, try reducing the lr to + # get us out of an oscilliation + if adaptive_lr==True: + classifier.lr.value=classifier.lr.value/2.0 + test_score = 0. + #cap the patience so we are allowed one more validation error + #calculation before aborting + patience = iter+validation_frequency+1 for x,y in test_batches: x_float=x/255.0 test_score += test_model(x_float,y) test_score /= len(test_batches) if verbose == True: - print ' validation error is going up, stopping now' + print ' validation error is going up, possibly stopping soon' print((' epoch %i, minibatch %i/%i, test error of best ' 'model %f %%') % (epoch, minibatch_index+1, n_minibatches, test_score*100.)) - break + - - if patience <= iter : - break - + if iter>patience: + print 'we have diverged' + break + end_time = time.clock() if verbose == True: @@ -391,17 +431,25 @@ 'obtained at iteration %i, with test performance %f %%') % (best_validation_loss * 100., best_iter, test_score*100.)) print ('The code ran for %f minutes' % ((end_time-start_time)/60.)) - print iter - return (best_validation_loss * 100.,test_score*100.,best_iter*batch_size,(end_time-start_time)/60) + print iter + + #save the model and the weights + numpy.savez('model.npy', config=configuration, W1=classifier.W1.value,W2=classifier.W2.value, b1=classifier.b1.value,b2=classifier.b2.value) + numpy.savez('results.npy',config=configuration,total_train_error_list=total_train_error_list,total_validation_error_list=total_validation_error_list,\ + learning_rate_list=learning_rate_list) + + return (best_training_error*100.0,best_validation_loss * 100.,test_score*100.,best_iter*batch_size,(end_time-start_time)/60) if __name__ == '__main__': mlp_full_mnist() def jobman_mlp_full_nist(state,channel): - (validation_error,test_error,nb_exemples,time)=mlp_full_nist(learning_rate=state.learning_rate,\ + (train_error,validation_error,test_error,nb_exemples,time)=mlp_full_nist(learning_rate=state.learning_rate,\ nb_max_exemples=state.nb_max_exemples,\ - nb_hidden=state.nb_hidden) + nb_hidden=state.nb_hidden,\ + adaptive_lr=state.adaptive_lr) + state.train_error=train_error state.validation_error=validation_error state.test_error=test_error state.nb_exemples=nb_exemples