--- a/deep/stacked_dae/nist_sda.py	Thu Mar 04 09:43:23 2010 -0500
+++ b/deep/stacked_dae/nist_sda.py	Thu Mar 04 20:43:21 2010 -0500
@@ -21,33 +21,35 @@
 import jobman, jobman.sql
 from pylearn.io import filetensor
 
-from utils import produit_croise_jobs
+from utils import produit_cartesien_jobs
 
 from sgd_optimization import SdaSgdOptimizer
 
-SERIES_AVAILABLE = False
-try:
-    from scalar_series import *
-    SERIES_AVAILABLE = True
-except ImportError:
-    print "Could not import Series"
+from ift6266.utils.scalar_series import *
+
+##############################################################################
+# GLOBALS
 
 TEST_CONFIG = False
 
 NIST_ALL_LOCATION = '/data/lisa/data/nist/by_class/all'
-
-JOBDB = 'postgres://ift6266h10@gershwin/ift6266h10_db/fsavard_sda2'
+JOBDB = 'postgres://ift6266h10@gershwin/ift6266h10_db/fsavard_sda4'
+EXPERIMENT_PATH = "ift6266.deep.stacked_dae.nist_sda.jobman_entrypoint"
 
 REDUCE_TRAIN_TO = None
 MAX_FINETUNING_EPOCHS = 1000
-REDUCE_EVERY = 1000 # number of minibatches before taking means for valid error etc.
+# number of minibatches before taking means for valid error etc.
+REDUCE_EVERY = 1000
+
 if TEST_CONFIG:
     REDUCE_TRAIN_TO = 1000
     MAX_FINETUNING_EPOCHS = 2
     REDUCE_EVERY = 10
 
-EXPERIMENT_PATH = "ift6266.deep.stacked_dae.nist_sda.jobman_entrypoint"
-
+# Possible values the hyperparameters can take. These are then
+# combined with produit_cartesien_jobs so we get a list of all
+# possible combinations, each one resulting in a job inserted
+# in the jobman DB.
 JOB_VALS = {'pretraining_lr': [0.1, 0.01],#, 0.001],#, 0.0001],
         'pretraining_epochs_per_layer': [10,20],
         'hidden_layers_sizes': [300,800],
@@ -58,30 +60,36 @@
         'num_hidden_layers':[2,3]}
 
 # Just useful for tests... minimal number of epochs
-DEFAULT_HP_NIST = DD({'finetuning_lr':0.01,
-                       'pretraining_lr':0.01,
-                       'pretraining_epochs_per_layer':1,
-                       'max_finetuning_epochs':1,
-                       'hidden_layers_sizes':1000,
+DEFAULT_HP_NIST = DD({'finetuning_lr':0.1,
+                       'pretraining_lr':0.1,
+                       'pretraining_epochs_per_layer':20,
+                       'max_finetuning_epochs':2,
+                       'hidden_layers_sizes':300,
                        'corruption_levels':0.2,
                        'minibatch_size':20,
-                       'reduce_train_to':1000,
-                       'num_hidden_layers':1})
+                       #'reduce_train_to':300,
+                       'num_hidden_layers':2})
 
+'''
+Function called by jobman upon launching each job
+Its path is the one given when inserting jobs:
+ift6266.deep.stacked_dae.nist_sda.jobman_entrypoint
+'''
 def jobman_entrypoint(state, channel):
+    # record mercurial versions of each package
     pylearn.version.record_versions(state,[theano,ift6266,pylearn])
     channel.save()
 
     workingdir = os.getcwd()
 
     print "Will load NIST"
-    sys.stdout.flush()
 
-    nist = NIST(20)
+    nist = NIST(minibatch_size=20)
 
     print "NIST loaded"
-    sys.stdout.flush()
 
+    # For test runs, we don't want to use the whole dataset so
+    # reduce it to fewer elements if asked to.
     rtt = None
     if state.has_key('reduce_train_to'):
         rtt = state['reduce_train_to']
@@ -89,7 +97,7 @@
         rtt = REDUCE_TRAIN_TO
 
     if rtt:
-        print "Reducing training set to ", rtt, " examples"
+        print "Reducing training set to "+str(rtt)+ " examples"
         nist.reduce_train_set(rtt)
 
     train,valid,test = nist.get_tvt()
@@ -98,17 +106,13 @@
     n_ins = 32*32
     n_outs = 62 # 10 digits, 26*2 (lower, capitals)
 
-    hls = state.hidden_layers_sizes
-    cl = state.corruption_levels
-    nhl = state.num_hidden_layers
-    state.hidden_layers_sizes = [hls] * nhl
-    state.corruption_levels = [cl] * nhl
+    # b,b',W for each hidden layer 
+    # + b,W of last layer (logreg)
+    numparams = state.num_hidden_layers * 3 + 2
+    series_mux = None
+    series_mux = create_series(workingdir, numparams)
 
-    # b,b',W for each hidden layer + b,W of last layer (logreg)
-    numparams = nhl * 3 + 2
-    series_mux = None
-    if SERIES_AVAILABLE:
-        series_mux = create_series(workingdir, numparams)
+    print "Creating optimizer with state, ", state
 
     optimizer = SdaSgdOptimizer(dataset=dataset, hyperparameters=state, \
                                     n_ins=n_ins, n_outs=n_outs,\
@@ -120,11 +124,10 @@
     optimizer.finetune()
     channel.save()
 
-    pylearn.version.record_versions(state,[theano,ift6266,pylearn])
-    channel.save()
-
     return channel.COMPLETE
 
+# These Series objects are used to save various statistics
+# during the training.
 def create_series(basedir, numparams):
     mux = SeriesMultiplexer()
 
@@ -146,8 +149,11 @@
 
     return mux
 
+# Perform insertion into the Postgre DB based on combination
+# of hyperparameter values above
+# (see comment for produit_cartesien_jobs() to know how it works)
 def jobman_insert_nist():
-    jobs = produit_croise_jobs(JOB_VALS)
+    jobs = produit_cartesien_jobs(JOB_VALS)
 
     db = jobman.sql.db(JOBDB)
     for job in jobs:
@@ -233,35 +239,6 @@
 
     raw_input("Press any key")
 
-# hp for hyperparameters
-def sgd_optimization_nist(hp=None, dataset_dir='/data/lisa/data/nist'):
-    global DEFAULT_HP_NIST
-    hp = hp and hp or DEFAULT_HP_NIST
-
-    print "Will load NIST"
-
-    import time
-    t1 = time.time()
-    nist = NIST(20, reduce_train_to=100)
-    t2 = time.time()
-
-    print "NIST loaded. time delta = ", t2-t1
-
-    train,valid,test = nist.get_tvt()
-    dataset = (train,valid,test)
-
-    print train[0][15]
-    print type(train[0][1])
-
-
-    print "Lengths train, valid, test: ", len(train[0]), len(valid[0]), len(test[0])
-
-    n_ins = 32*32
-    n_outs = 62 # 10 digits, 26*2 (lower, capitals)
-
-    optimizer = SdaSgdOptimizer(dataset, hp, n_ins, n_outs, input_divider=255.0)
-    optimizer.train()
-
 if __name__ == '__main__':
 
     import sys
@@ -275,11 +252,9 @@
         jobman_insert_nist()
 
     elif len(args) > 0 and args[0] == 'test_jobman_entrypoint':
-        chanmock = DD({'COMPLETE':0})
+        chanmock = DD({'COMPLETE':0,'save':(lambda:None)})
         jobman_entrypoint(DEFAULT_HP_NIST, chanmock)
 
-    elif len(args) > 0 and args[0] == 'estimate':
-        estimate_total_time()
     else:
-        sgd_optimization_nist()
+        print "Bad arguments"