# HG changeset patch # User fsavard # Date 1266863905 18000 # Node ID 7d8366fb90bfaeab0aea04aefaf1d8ff57b60c3f # Parent 128507ac4edf70d451cf0bdb690b27933da5635f Ajouté des __init__.py dans l'arborescence pour que les scripts puissent être utilisés avec des paths pour jobman, et fait pas mal de modifs dans stacked_dae pour pouvoir réutiliser le travail fait pour des tests où le pretraining est le même. diff -r 128507ac4edf -r 7d8366fb90bf scripts/stacked_dae/mnist_sda.py --- a/scripts/stacked_dae/mnist_sda.py Sun Feb 21 17:30:38 2010 -0600 +++ b/scripts/stacked_dae/mnist_sda.py Mon Feb 22 13:38:25 2010 -0500 @@ -9,7 +9,7 @@ import theano.tensor as T from theano.tensor.shared_randomstreams import RandomStreams -from stacked_dae import sgd_optimization +from sgd_optimization import SdaSgdOptimizer import cPickle, gzip from jobman import DD @@ -31,12 +31,14 @@ 'pretraining_lr':pretrain_lr, 'pretraining_epochs_per_layer':pretraining_epochs, 'max_finetuning_epochs':training_epochs, - 'hidden_layers_sizes':[1000,1000,1000], - 'corruption_levels':[0.2,0.2,0.2], + 'hidden_layers_sizes':[100], + 'corruption_levels':[0.2], 'minibatch_size':20}) - sgd_optimization(dataset, hyperparameters, n_ins, n_outs) + optimizer = SdaSgdOptimizer(dataset, hyperparameters, n_ins, n_outs) + optimizer.pretrain() + optimizer.finetune() if __name__ == '__main__': - sgd_optimization_mnist() + sgd_optimization_mnist(dataset=MNIST_LOCATION) diff -r 128507ac4edf -r 7d8366fb90bf scripts/stacked_dae/nist_sda.py --- a/scripts/stacked_dae/nist_sda.py Sun Feb 21 17:30:38 2010 -0600 +++ b/scripts/stacked_dae/nist_sda.py Mon Feb 22 13:38:25 2010 -0500 @@ -6,47 +6,135 @@ import time import theano.tensor as T from theano.tensor.shared_randomstreams import RandomStreams +import copy +import sys import os.path -from sgd_optimization import sgd_optimization +from sgd_optimization import SdaSgdOptimizer from jobman import DD +import jobman, jobman.sql from pylearn.io import filetensor from utils import produit_croise_jobs +TEST_CONFIG = True + NIST_ALL_LOCATION = '/data/lisa/data/nist/by_class/all' +JOBDB = 'postgres://ift6266h10@gershwin/ift6266h10_db/' +REDUCE_TRAIN_TO = None +MAX_FINETUNING_EPOCHS = 1000 +if TEST_CONFIG: + JOBDB = 'postgres://ift6266h10@gershwin/ift6266h10_sandbox_db/' + REDUCE_TRAIN_TO = 1000 + MAX_FINETUNING_EPOCHS = 2 + +JOBDB_JOBS = JOBDB + 'fsavard_sda1_jobs' +JOBDB_RESULTS = JOBDB + 'fsavard_sda1_results' +EXPERIMENT_PATH = "ift6266.scripts.stacked_dae.nist_sda.jobman_entrypoint" + +# There used to be +# 'finetuning_lr': [0.00001, 0.0001, 0.001, 0.01, 0.1] +# and +# 'num_hidden_layers':[1,2,3] +# but this is now handled by a special mechanism in SgdOptimizer +# to reuse intermediate results (for the same training of lower layers, +# we can test many finetuning_lr) +JOB_VALS = {'pretraining_lr': [0.1, 0.01, 0.001],#, 0.0001], + 'pretraining_epochs_per_layer': [10,20], + 'hidden_layers_sizes': [300,800], + 'corruption_levels': [0.1,0.2], + 'minibatch_size': [20], + 'max_finetuning_epochs':[MAX_FINETUNING_EPOCHS]} +FINETUNING_LR_VALS = [0.1, 0.01, 0.001]#, 0.0001] +NUM_HIDDEN_LAYERS_VALS = [1,2,3] + # Just useful for tests... minimal number of epochs -DEFAULT_HP_NIST = DD({'finetuning_lr':0.1, - 'pretraining_lr':0.1, +DEFAULT_HP_NIST = DD({'finetuning_lr':0.01, + 'pretraining_lr':0.01, 'pretraining_epochs_per_layer':1, 'max_finetuning_epochs':1, - 'hidden_layers_sizes':[1000,1000], - 'corruption_levels':[0.2,0.2], + 'hidden_layers_sizes':[1000], + 'corruption_levels':[0.2], 'minibatch_size':20}) -def jobman_entrypoint_nist(state, channel): - sgd_optimization_nist(state) +def jobman_entrypoint(state, channel): + state = copy.copy(state) + + print "Will load NIST" + nist = NIST(20) + print "NIST loaded" + + rtt = None + if state.has_key('reduce_train_to'): + rtt = state['reduce_train_to'] + elif REDUCE_TRAIN_TO: + rtt = REDUCE_TRAIN_TO + + if rtt: + print "Reducing training set to ", rtt, " examples" + nist.reduce_train_set(rtt) + + train,valid,test = nist.get_tvt() + dataset = (train,valid,test) + + n_ins = 32*32 + n_outs = 62 # 10 digits, 26*2 (lower, capitals) + + db = jobman.sql.db(JOBDB_RESULTS) + optimizer = SdaSgdOptimizer(dataset, state, n_ins, n_outs,\ + input_divider=255.0, job_tree=True, results_db=db, \ + experiment=EXPERIMENT_PATH, \ + finetuning_lr_to_try=FINETUNING_LR_VALS, \ + num_hidden_layers_to_try=NUM_HIDDEN_LAYERS_VALS) + optimizer.train() + + return channel.COMPLETE + +def estimate_pretraining_time(job): + job = DD(job) + # time spent on pretraining estimated as O(n^2) where n=num hidens + # no need to multiply by num_hidden_layers, as results from num=1 + # is reused for num=2, or 3, so in the end we get the same time + # as if we were training 3 times a single layer + # constants: + # - 20 mins to pretrain a layer with 1000 units (per 1 epoch) + # - 12 mins to finetune (per 1 epoch) + # basically the job_tree trick gives us a 5 times speedup on the + # pretraining time due to reusing for finetuning_lr + # and gives us a second x2 speedup for reusing previous layers + # to explore num_hidden_layers + return (job.pretraining_epochs_per_layer * 20 / (1000.0*1000) \ + * job.hidden_layer_sizes * job.hidden_layer_sizes) + +def estimate_total_time(): + jobs = produit_croise_jobs(JOB_VALS) + sumtime = 0.0 + sum_without = 0.0 + for job in jobs: + sumtime += estimate_pretraining_time(job) + # 12 mins per epoch * 30 epochs + # 5 finetuning_lr per pretraining combination + sum_without = (12*20*len(jobs) + sumtime*2) * len(FINETUNING_LR_VALS) + sumtime += len(FINETUNING_LR_VALS) * len(jobs) * 12 * 20 + print "num jobs=", len(jobs) + print "estimate", sumtime/60, " hours" + print "estimate without tree optimization", sum_without/60, "ratio", sumtime / sum_without def jobman_insert_nist(): - vals = {'finetuning_lr': [0.00001, 0.0001, 0.001, 0.01, 0.1], - 'pretraining_lr': [0.00001, 0.0001, 0.001, 0.01, 0.1], - 'pretraining_epochs_per_layer': [2,5,20], - 'hidden_layer_sizes': [100,300,1000], - 'num_hidden_layers':[1,2,3], - 'corruption_levels': [0.1,0.2,0.4], - 'minibatch_size': [5,20,100]} + jobs = produit_croise_jobs(JOB_VALS) - jobs = produit_croise_jobs(vals) - + db = jobman.sql.db(JOBDB_JOBS) for job in jobs: - insert_job(job) + job.update({jobman.sql.EXPERIMENT: EXPERIMENT_PATH}) + jobman.sql.insert_dict(job, db) + print "inserted" class NIST: - def __init__(self, minibatch_size, basepath=None): + def __init__(self, minibatch_size, basepath=None, reduce_train_to=None): global NIST_ALL_LOCATION self.minibatch_size = minibatch_size @@ -61,8 +149,9 @@ self.load_train_test() self.valid = [[], []] - #self.split_train_valid() - + self.split_train_valid() + if reduce_train_to: + self.reduce_train_set(reduce_train_to) def get_tvt(self): return self.train, self.valid, self.test @@ -84,6 +173,15 @@ pair[i] = filetensor.read(f) f.close() + def reduce_train_set(self, max): + self.train[0] = self.train[0][:max] + self.train[1] = self.train[1][:max] + + if max < len(self.test[0]): + for ar in (self.test, self.valid): + ar[0] = ar[0][:max] + ar[1] = ar[1][:max] + def split_train_valid(self): test_len = len(self.test[0]) @@ -121,7 +219,7 @@ import time t1 = time.time() - nist = NIST(20) + nist = NIST(20, reduce_train_to=100) t2 = time.time() print "NIST loaded. time delta = ", t2-t1 @@ -129,12 +227,17 @@ train,valid,test = nist.get_tvt() dataset = (train,valid,test) - print "Lenghts train, valid, test: ", len(train[0]), len(valid[0]), len(test[0]) + print train[0][15] + print type(train[0][1]) + + + print "Lengths train, valid, test: ", len(train[0]), len(valid[0]), len(test[0]) n_ins = 32*32 n_outs = 62 # 10 digits, 26*2 (lower, capitals) - sgd_optimization(dataset, hp, n_ins, n_outs) + optimizer = SdaSgdOptimizer(dataset, hp, n_ins, n_outs, input_divider=255.0) + optimizer.train() if __name__ == '__main__': @@ -145,6 +248,17 @@ if len(args) > 0 and args[0] == 'load_nist': test_load_nist() + elif len(args) > 0 and args[0] == 'jobman_insert': + jobman_insert_nist() + elif len(args) > 0 and args[0] == 'test_job_tree': + # dont forget to comment out sql.inserts and make reduce_train_to=100 + print "TESTING JOB TREE" + chanmock = {'COMPLETE':0} + hp = copy.copy(DEFAULT_HP_NIST) + hp.update({'reduce_train_to':100}) + jobman_entrypoint(hp, chanmock) + elif len(args) > 0 and args[0] == 'estimate': + estimate_total_time() else: sgd_optimization_nist() diff -r 128507ac4edf -r 7d8366fb90bf scripts/stacked_dae/sgd_optimization.py --- a/scripts/stacked_dae/sgd_optimization.py Sun Feb 21 17:30:38 2010 -0600 +++ b/scripts/stacked_dae/sgd_optimization.py Mon Feb 22 13:38:25 2010 -0500 @@ -1,165 +1,270 @@ #!/usr/bin/python # coding: utf-8 -# Generic SdA optimization loop, adapted slightly from the deeplearning.net tutorial +# Generic SdA optimization loop, adapted from the deeplearning.net tutorial import numpy import theano import time import theano.tensor as T +import copy +import sys from jobman import DD +import jobman, jobman.sql from stacked_dae import SdA -def sgd_optimization(dataset, hyperparameters, n_ins, n_outs): - hp = hyperparameters +def shared_dataset(data_xy): + data_x, data_y = data_xy + #shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX)) + #shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX)) + #shared_y = T.cast(shared_y, 'int32') + shared_x = theano.shared(data_x) + shared_y = theano.shared(data_y) + return shared_x, shared_y - printout_frequency = 1000 - - train_set, valid_set, test_set = dataset +class SdaSgdOptimizer: + def __init__(self, dataset, hyperparameters, n_ins, n_outs, input_divider=1.0,\ + job_tree=False, results_db=None,\ + experiment="",\ + num_hidden_layers_to_try=[1,2,3], \ + finetuning_lr_to_try=[0.1, 0.01, 0.001, 0.0001, 0.00001]): - def shared_dataset(data_xy): - data_x, data_y = data_xy - shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX)) - shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX)) - return shared_x, T.cast(shared_y, 'int32') + self.dataset = dataset + self.hp = copy.copy(hyperparameters) + self.n_ins = n_ins + self.n_outs = n_outs + self.input_divider = numpy.asarray(input_divider, dtype=theano.config.floatX) - test_set_x, test_set_y = shared_dataset(test_set) - valid_set_x, valid_set_y = shared_dataset(valid_set) - train_set_x, train_set_y = shared_dataset(train_set) + self.job_tree = job_tree + self.results_db = results_db + self.experiment = experiment + if self.job_tree: + assert(not results_db is None) + # these hp should not be there, so we insert default values + # we use 3 hidden layers as we'll iterate through 1,2,3 + self.hp.finetuning_lr = 0.1 # dummy value, will be replaced anyway + cl = self.hp.corruption_levels + nh = self.hp.hidden_layers_sizes + self.hp.corruption_levels = [cl,cl,cl] + self.hp.hidden_layers_sizes = [nh,nh,nh] + + self.num_hidden_layers_to_try = num_hidden_layers_to_try + self.finetuning_lr_to_try = finetuning_lr_to_try + + self.printout_frequency = 1000 - # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.value.shape[0] / hp.minibatch_size - n_valid_batches = valid_set_x.value.shape[0] / hp.minibatch_size - n_test_batches = test_set_x.value.shape[0] / hp.minibatch_size + self.rng = numpy.random.RandomState(1234) + + self.init_datasets() + self.init_classifier() + + def init_datasets(self): + print "init_datasets" + train_set, valid_set, test_set = self.dataset + self.test_set_x, self.test_set_y = shared_dataset(test_set) + self.valid_set_x, self.valid_set_y = shared_dataset(valid_set) + self.train_set_x, self.train_set_y = shared_dataset(train_set) + + # compute number of minibatches for training, validation and testing + self.n_train_batches = self.train_set_x.value.shape[0] / self.hp.minibatch_size + self.n_valid_batches = self.valid_set_x.value.shape[0] / self.hp.minibatch_size + self.n_test_batches = self.test_set_x.value.shape[0] / self.hp.minibatch_size - # allocate symbolic variables for the data - index = T.lscalar() # index to a [mini]batch - - # construct the stacked denoising autoencoder class - classifier = SdA( train_set_x=train_set_x, train_set_y = train_set_y,\ - batch_size = hp.minibatch_size, n_ins= n_ins, \ - hidden_layers_sizes = hp.hidden_layers_sizes, n_outs=10, \ - corruption_levels = hp.corruption_levels,\ - rng = numpy.random.RandomState(1234),\ - pretrain_lr = hp.pretraining_lr, finetune_lr = hp.finetuning_lr ) + def init_classifier(self): + print "Constructing classifier" + # construct the stacked denoising autoencoder class + self.classifier = SdA( \ + train_set_x= self.train_set_x, \ + train_set_y = self.train_set_y,\ + batch_size = self.hp.minibatch_size, \ + n_ins= self.n_ins, \ + hidden_layers_sizes = self.hp.hidden_layers_sizes, \ + n_outs = self.n_outs, \ + corruption_levels = self.hp.corruption_levels,\ + rng = self.rng,\ + pretrain_lr = self.hp.pretraining_lr, \ + finetune_lr = self.hp.finetuning_lr,\ + input_divider = self.input_divider ) - printout_acc = 0.0 + def train(self): + self.pretrain() + if not self.job_tree: + # if job_tree is True, finetuning was already performed + self.finetune() + + def pretrain(self): + print "STARTING PRETRAINING" - start_time = time.clock() - ## Pre-train layer-wise - for i in xrange(classifier.n_layers): - # go through pretraining epochs - for epoch in xrange(hp.pretraining_epochs_per_layer): - # go through the training set - for batch_index in xrange(n_train_batches): - c = classifier.pretrain_functions[i](batch_index) + printout_acc = 0.0 + last_error = 0.0 + + start_time = time.clock() + ## Pre-train layer-wise + for i in xrange(self.classifier.n_layers): + # go through pretraining epochs + for epoch in xrange(self.hp.pretraining_epochs_per_layer): + # go through the training set + for batch_index in xrange(self.n_train_batches): + c = self.classifier.pretrain_functions[i](batch_index) - print c + printout_acc += c / self.printout_frequency + if (batch_index+1) % self.printout_frequency == 0: + print batch_index, "reconstruction cost avg=", printout_acc + last_error = printout_acc + printout_acc = 0.0 + + print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c + + self.job_splitter(i+1, time.clock()-start_time, last_error) + + end_time = time.clock() + + print ('Pretraining took %f minutes' %((end_time-start_time)/60.)) + + # Save time by reusing intermediate results + def job_splitter(self, current_pretraining_layer, pretraining_time, last_error): + + state_copy = None + original_classifier = None + + if self.job_tree and current_pretraining_layer in self.num_hidden_layers_to_try: + for lr in self.finetuning_lr_to_try: + sys.stdout.flush() + sys.stderr.flush() + + state_copy = copy.copy(self.hp) - printout_acc += c / printout_frequency - if (batch_index+1) % printout_frequency == 0: - print batch_index, "reconstruction cost avg=", printout_acc - printout_acc = 0.0 - - print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),c - - end_time = time.clock() + self.hp.update({'num_hidden_layers':current_pretraining_layer, \ + 'finetuning_lr':lr,\ + 'pretraining_time':pretraining_time,\ + 'last_reconstruction_error':last_error}) - print ('Pretraining took %f minutes' %((end_time-start_time)/60.)) - # Fine-tune the entire model + original_classifier = self.classifier + print "ORIGINAL CLASSIFIER MEANS",original_classifier.get_params_means() + self.classifier = SdA.copy_reusing_lower_layers(original_classifier, current_pretraining_layer, new_finetuning_lr=lr) + + self.finetune() + + self.insert_finished_job() + + print "NEW CLASSIFIER MEANS AFTERWARDS",self.classifier.get_params_means() + print "ORIGINAL CLASSIFIER MEANS AFTERWARDS",original_classifier.get_params_means() + self.classifier = original_classifier + self.hp = state_copy + + def insert_finished_job(self): + job = copy.copy(self.hp) + job[jobman.sql.STATUS] = jobman.sql.DONE + job[jobman.sql.EXPERIMENT] = self.experiment - minibatch_size = hp.minibatch_size + # don,t try to store arrays in db + job['hidden_layers_sizes'] = job.hidden_layers_sizes[0] + job['corruption_levels'] = job.corruption_levels[0] + + print "Will insert finished job", job + jobman.sql.insert_dict(jobman.flatten(job), self.results_db) + + def finetune(self): + print "STARTING FINETUNING" - # create a function to compute the mistakes that are made by the model - # on the validation set, or testing set - test_model = theano.function([index], classifier.errors, - givens = { - classifier.x: test_set_x[index*minibatch_size:(index+1)*minibatch_size], - classifier.y: test_set_y[index*minibatch_size:(index+1)*minibatch_size]}) + index = T.lscalar() # index to a [mini]batch + minibatch_size = self.hp.minibatch_size - validate_model = theano.function([index], classifier.errors, - givens = { - classifier.x: valid_set_x[index*minibatch_size:(index+1)*minibatch_size], - classifier.y: valid_set_y[index*minibatch_size:(index+1)*minibatch_size]}) + # create a function to compute the mistakes that are made by the model + # on the validation set, or testing set + test_model = theano.function([index], self.classifier.errors, + givens = { + self.classifier.x: self.test_set_x[index*minibatch_size:(index+1)*minibatch_size] / self.input_divider, + self.classifier.y: self.test_set_y[index*minibatch_size:(index+1)*minibatch_size]}) + + validate_model = theano.function([index], self.classifier.errors, + givens = { + self.classifier.x: self.valid_set_x[index*minibatch_size:(index+1)*minibatch_size] / self.input_divider, + self.classifier.y: self.valid_set_y[index*minibatch_size:(index+1)*minibatch_size]}) - # early-stopping parameters - patience = 10000 # look as this many examples regardless - patience_increase = 2. # wait this much longer when a new best is - # found - improvement_threshold = 0.995 # a relative improvement of this much is - # considered significant - validation_frequency = min(n_train_batches, patience/2) - # go through this many - # minibatche before checking the network - # on the validation set; in this case we - # check every epoch + # early-stopping parameters + patience = 10000 # look as this many examples regardless + patience_increase = 2. # wait this much longer when a new best is + # found + improvement_threshold = 0.995 # a relative improvement of this much is + # considered significant + validation_frequency = min(self.n_train_batches, patience/2) + # go through this many + # minibatche before checking the network + # on the validation set; in this case we + # check every epoch - best_params = None - best_validation_loss = float('inf') - test_score = 0. - start_time = time.clock() + best_params = None + best_validation_loss = float('inf') + test_score = 0. + start_time = time.clock() - done_looping = False - epoch = 0 + done_looping = False + epoch = 0 - printout_acc = 0.0 + printout_acc = 0.0 - print "----- START FINETUNING -----" + if not self.hp.has_key('max_finetuning_epochs'): + self.hp.max_finetuning_epochs = 1000 - while (epoch < hp.max_finetuning_epochs) and (not done_looping): - epoch = epoch + 1 - for minibatch_index in xrange(n_train_batches): + while (epoch < self.hp.max_finetuning_epochs) and (not done_looping): + epoch = epoch + 1 + for minibatch_index in xrange(self.n_train_batches): - cost_ij = classifier.finetune(minibatch_index) - iter = epoch * n_train_batches + minibatch_index + cost_ij = self.classifier.finetune(minibatch_index) + iter = epoch * self.n_train_batches + minibatch_index - printout_acc += cost_ij / float(printout_frequency * minibatch_size) - if (iter+1) % printout_frequency == 0: - print iter, "cost avg=", printout_acc - printout_acc = 0.0 + printout_acc += cost_ij / float(self.printout_frequency * minibatch_size) + if (iter+1) % self.printout_frequency == 0: + print iter, "cost avg=", printout_acc + printout_acc = 0.0 - if (iter+1) % validation_frequency == 0: - - validation_losses = [validate_model(i) for i in xrange(n_valid_batches)] - this_validation_loss = numpy.mean(validation_losses) - print('epoch %i, minibatch %i/%i, validation error %f %%' % \ - (epoch, minibatch_index+1, n_train_batches, \ - this_validation_loss*100.)) + if (iter+1) % validation_frequency == 0: + + validation_losses = [validate_model(i) for i in xrange(self.n_valid_batches)] + this_validation_loss = numpy.mean(validation_losses) + print('epoch %i, minibatch %i/%i, validation error %f %%' % \ + (epoch, minibatch_index+1, self.n_train_batches, \ + this_validation_loss*100.)) - # if we got the best validation score until now - if this_validation_loss < best_validation_loss: + # if we got the best validation score until now + if this_validation_loss < best_validation_loss: - #improve patience if loss improvement is good enough - if this_validation_loss < best_validation_loss * \ - improvement_threshold : - patience = max(patience, iter * patience_increase) + #improve patience if loss improvement is good enough + if this_validation_loss < best_validation_loss * \ + improvement_threshold : + patience = max(patience, iter * patience_increase) - # save best validation score and iteration number - best_validation_loss = this_validation_loss - best_iter = iter + # save best validation score and iteration number + best_validation_loss = this_validation_loss + best_iter = iter - # test it on the test set - test_losses = [test_model(i) for i in xrange(n_test_batches)] - test_score = numpy.mean(test_losses) - print((' epoch %i, minibatch %i/%i, test error of best ' - 'model %f %%') % - (epoch, minibatch_index+1, n_train_batches, - test_score*100.)) + # test it on the test set + test_losses = [test_model(i) for i in xrange(self.n_test_batches)] + test_score = numpy.mean(test_losses) + print((' epoch %i, minibatch %i/%i, test error of best ' + 'model %f %%') % + (epoch, minibatch_index+1, self.n_train_batches, + test_score*100.)) - if patience <= iter : + if patience <= iter : done_looping = True break - end_time = time.clock() - print(('Optimization complete with best validation score of %f %%,' - 'with test performance %f %%') % - - (best_validation_loss * 100., test_score*100.)) - print ('The code ran for %f minutes' % ((end_time-start_time)/60.)) + end_time = time.clock() + self.hp.update({'finetuning_time':end_time-start_time,\ + 'best_validation_error':best_validation_loss,\ + 'test_score':test_score, + 'num_finetuning_epochs':epoch}) + print(('Optimization complete with best validation score of %f %%,' + 'with test performance %f %%') % + (best_validation_loss * 100., test_score*100.)) + print ('The finetuning ran for %f minutes' % ((end_time-start_time)/60.)) + diff -r 128507ac4edf -r 7d8366fb90bf scripts/stacked_dae/stacked_dae.py --- a/scripts/stacked_dae/stacked_dae.py Sun Feb 21 17:30:38 2010 -0600 +++ b/scripts/stacked_dae/stacked_dae.py Mon Feb 22 13:38:25 2010 -0500 @@ -6,6 +6,9 @@ import time import theano.tensor as T from theano.tensor.shared_randomstreams import RandomStreams +import copy + +from utils import update_locals class LogisticRegression(object): def __init__(self, input, n_in, n_out): @@ -140,13 +143,16 @@ class SdA(object): def __init__(self, train_set_x, train_set_y, batch_size, n_ins, hidden_layers_sizes, n_outs, - corruption_levels, rng, pretrain_lr, finetune_lr): - + corruption_levels, rng, pretrain_lr, finetune_lr, input_divider=1.0): + update_locals(self, locals()) + self.layers = [] self.pretrain_functions = [] self.params = [] self.n_layers = len(hidden_layers_sizes) + self.input_divider = numpy.asarray(input_divider, dtype=theano.config.floatX) + if len(hidden_layers_sizes) < 1 : raiseException (' You must have at least one hidden layer ') @@ -200,7 +206,7 @@ update_fn = theano.function([index], dA_layer.cost, \ updates = updates, givens = { - self.x : train_set_x[index*batch_size:(index+1)*batch_size]}) + self.x : train_set_x[index*batch_size:(index+1)*batch_size] / self.input_divider}) # collect this function into a list self.pretrain_functions += [update_fn] @@ -225,7 +231,7 @@ self.finetune = theano.function([index], cost, updates = updates, givens = { - self.x : train_set_x[index*batch_size:(index+1)*batch_size], + self.x : train_set_x[index*batch_size:(index+1)*batch_size]/self.input_divider, self.y : train_set_y[index*batch_size:(index+1)*batch_size]} ) # symbolic variable that points to the number of errors made on the @@ -233,23 +239,49 @@ self.errors = self.logLayer.errors(self.y) + @classmethod + def copy_reusing_lower_layers(cls, obj, num_hidden_layers, new_finetuning_lr=None): + assert(num_hidden_layers <= obj.n_layers) + + if not new_finetuning_lr: + new_finetuning_lr = obj.finetune_lr + + new_sda = cls(train_set_x= obj.train_set_x, \ + train_set_y = obj.train_set_y,\ + batch_size = obj.batch_size, \ + n_ins= obj.n_ins, \ + hidden_layers_sizes = obj.hidden_layers_sizes[:num_hidden_layers], \ + n_outs = obj.n_outs, \ + corruption_levels = obj.corruption_levels[:num_hidden_layers],\ + rng = obj.rng,\ + pretrain_lr = obj.pretrain_lr, \ + finetune_lr = new_finetuning_lr, \ + input_divider = obj.input_divider ) + + # new_sda.layers contains only the hidden layers actually + for i, layer in enumerate(new_sda.layers): + original_layer = obj.layers[i] + for p1,p2 in zip(layer.params, original_layer.params): + p1.value = p2.value.copy() + + return new_sda + + def get_params_copy(self): + return copy.deepcopy(self.params) + + def set_params_from_copy(self, copy): + # We don't want to replace the var, as the functions have pointers in there + # We only want to replace values. + for i, p in enumerate(self.params): + p.value = copy[i].value + + def get_params_means(self): + s = [] + for p in self.params: + s.append(numpy.mean(p.value)) + return s + if __name__ == '__main__': import sys args = sys.argv[1:] - if len(args) < 1: - print "Options: mnist, jobman_add, load_nist" - sys.exit(0) - - if args[0] == "jobman_add": - jobman_add() - elif args[0] == "mnist": - sgd_optimization_mnist(dataset=MNIST_LOCATION) - elif args[0] == "load_nist": - load_nist_test() - elif args[0] == "nist": - sgd_optimization_nist() - elif args[0] == "pc": - test_produit_croise_jobs() - - diff -r 128507ac4edf -r 7d8366fb90bf scripts/stacked_dae/utils.py --- a/scripts/stacked_dae/utils.py Sun Feb 21 17:30:38 2010 -0600 +++ b/scripts/stacked_dae/utils.py Mon Feb 22 13:38:25 2010 -0500 @@ -2,6 +2,12 @@ from jobman import DD +# from pylearn codebase +def update_locals(obj, dct): + if 'self' in dct: + del dct['self'] + obj.__dict__.update(dct) + def produit_croise_jobs(val_dict): job_list = [DD()] all_keys = val_dict.keys() @@ -23,3 +29,29 @@ print produit_croise_jobs(vals) +# taken from http://stackoverflow.com/questions/276052/how-to-get-current-cpu-and-ram-usage-in-python +"""Simple module for getting amount of memory used by a specified user's +processes on a UNIX system. +It uses UNIX ps utility to get the memory usage for a specified username and +pipe it to awk for summing up per application memory usage and return the total. +Python's Popen() from subprocess module is used for spawning ps and awk. + +""" + +import subprocess + +class MemoryMonitor(object): + + def __init__(self, username): + """Create new MemoryMonitor instance.""" + self.username = username + + def usage(self): + """Return int containing memory used by user's processes.""" + self.process = subprocess.Popen("ps -u %s -o rss | awk '{sum+=$1} END {print sum}'" % self.username, + shell=True, + stdout=subprocess.PIPE, + ) + self.stdout_list = self.process.communicate()[0].split('\n') + return int(self.stdout_list[0]) +