Mercurial > ift6266

comparison deep/convolutional_dae/salah_exp/nist_csda.py @ 358:31641a84e0ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial commit for the experimental setup of the denoising convolutional network
author humel
date Thu, 22 Apr 2010 00:49:42 -0400
parents
children c05680f8c92f
comparison
equal deleted inserted replaced
357:9a7b74927f7d 358:31641a84e0ae
1 #!/usr/bin/python
2 # coding: utf-8
3
4 import ift6266
5 import pylearn
6
7 import numpy
8 import theano
9 import time
10
11 import pylearn.version
12 import theano.tensor as T
13 from theano.tensor.shared_randomstreams import RandomStreams
14
15 import copy
16 import sys
17 import os
18 import os.path
19
20 from jobman import DD
21 import jobman, jobman.sql
22 from pylearn.io import filetensor
23
24 from utils import produit_cartesien_jobs
25 from copy import copy
26
27 from sgd_optimization_new import CSdASgdOptimizer
28
29 #from ift6266.utils.scalar_series import *
30 from ift6266.utils.seriestables import *
31 import tables
32
33 from ift6266 import datasets
34 from config import *
35
36 '''
37 Function called by jobman upon launching each job
38 Its path is the one given when inserting jobs: see EXPERIMENT_PATH
39 '''
40 def jobman_entrypoint(state, channel):
41 # record mercurial versions of each package
42 pylearn.version.record_versions(state,[theano,ift6266,pylearn])
43 # TODO: remove this, bad for number of simultaneous requests on DB
44 channel.save()
45
46 # For test runs, we don't want to use the whole dataset so
47 # reduce it to fewer elements if asked to.
48 rtt = None
49 #REDUCE_TRAIN_TO = 40000
50 if state.has_key('reduce_train_to'):
51 rtt = state['reduce_train_to']
52 elif REDUCE_TRAIN_TO:
53 rtt = REDUCE_TRAIN_TO
54
55 if state.has_key('decrease_lr'):
56 decrease_lr = state['decrease_lr']
57 else :
58 decrease_lr = 0
59
60 n_ins = 32*32
61 n_outs = 62 # 10 digits, 26*2 (lower, capitals)
62
63 examples_per_epoch = 100000#NIST_ALL_TRAIN_SIZE
64
65 #To be sure variables will not be only in the if statement
66 PATH = ''
67 nom_reptrain = ''
68 nom_serie = ""
69 if state['pretrain_choice'] == 0:
70 nom_serie="series_NIST.h5"
71 elif state['pretrain_choice'] == 1:
72 nom_serie="series_P07.h5"
73
74 series = create_series(state.num_hidden_layers,nom_serie)
75
76
77 print "Creating optimizer with state, ", state
78
79 optimizer = CSdASgdOptimizer(dataset=datasets.nist_all(),
80 hyperparameters=state, \
81 n_ins=n_ins, n_outs=n_outs,\
82 examples_per_epoch=examples_per_epoch, \
83 series=series,
84 max_minibatches=rtt)
85
86 parameters=[]
87 #Number of files of P07 used for pretraining
88 nb_file=0
89 if state['pretrain_choice'] == 0:
90 print('\n\tpretraining with NIST\n')
91 optimizer.pretrain(datasets.nist_all())
92 elif state['pretrain_choice'] == 1:
93 #To know how many file will be used during pretraining
94 nb_file = int(state['pretraining_epochs_per_layer'])
95 state['pretraining_epochs_per_layer'] = 1 #Only 1 time over the dataset
96 if nb_file >=100:
97 sys.exit("The code does not support this much pretraining epoch (99 max with P07).\n"+
98 "You have to correct the code (and be patient, P07 is huge !!)\n"+
99 "or reduce the number of pretraining epoch to run the code (better idea).\n")
100 print('\n\tpretraining with P07')
101 optimizer.pretrain(datasets.nist_P07(min_file=0,max_file=nb_file))
102 channel.save()
103
104 #Set some of the parameters used for the finetuning
105 if state.has_key('finetune_set'):
106 finetune_choice=state['finetune_set']
107 else:
108 finetune_choice=FINETUNE_SET
109
110 if state.has_key('max_finetuning_epochs'):
111 max_finetune_epoch_NIST=state['max_finetuning_epochs']
112 else:
113 max_finetune_epoch_NIST=MAX_FINETUNING_EPOCHS
114
115 if state.has_key('max_finetuning_epochs_P07'):
116 max_finetune_epoch_P07=state['max_finetuning_epochs_P07']
117 else:
118 max_finetune_epoch_P07=max_finetune_epoch_NIST
119
120 #Decide how the finetune is done
121
122 if finetune_choice == 0:
123 print('\n\n\tfinetune with NIST\n\n')
124 optimizer.reload_parameters('params_pretrain.txt')
125 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,decrease=decrease_lr)
126 channel.save()
127 if finetune_choice == 1:
128 print('\n\n\tfinetune with P07\n\n')
129 optimizer.reload_parameters('params_pretrain.txt')
130 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=0,decrease=decrease_lr)
131 channel.save()
132 if finetune_choice == 2:
133 print('\n\n\tfinetune with P07 followed by NIST\n\n')
134 optimizer.reload_parameters('params_pretrain.txt')
135 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=20,decrease=decrease_lr)
136 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=21,decrease=decrease_lr)
137 channel.save()
138 if finetune_choice == 3:
139 print('\n\n\tfinetune with NIST only on the logistic regression on top (but validation on P07).\n\
140 All hidden units output are input of the logistic regression\n\n')
141 optimizer.reload_parameters('params_pretrain.txt')
142 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,special=1,decrease=decrease_lr)
143
144
145 if finetune_choice==-1:
146 print('\nSERIE OF 4 DIFFERENT FINETUNINGS')
147 print('\n\n\tfinetune with NIST\n\n')
148 sys.stdout.flush()
149 optimizer.reload_parameters('params_pretrain.txt')
150 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,decrease=decrease_lr)
151 channel.save()
152 print('\n\n\tfinetune with P07\n\n')
153 sys.stdout.flush()
154 optimizer.reload_parameters('params_pretrain.txt')
155 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=0,decrease=decrease_lr)
156 channel.save()
157 print('\n\n\tfinetune with P07 (done earlier) followed by NIST (written here)\n\n')
158 sys.stdout.flush()
159 optimizer.reload_parameters('params_finetune_P07.txt')
160 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=21,decrease=decrease_lr)
161 channel.save()
162 print('\n\n\tfinetune with NIST only on the logistic regression on top.\n\
163 All hidden units output are input of the logistic regression\n\n')
164 sys.stdout.flush()
165 optimizer.reload_parameters('params_pretrain.txt')
166 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,special=1,decrease=decrease_lr)
167 channel.save()
168
169 channel.save()
170
171 return channel.COMPLETE
172
173 # These Series objects are used to save various statistics
174 # during the training.
175 def create_series(num_hidden_layers, nom_serie):
176
177 # Replace series we don't want to save with DummySeries, e.g.
178 # series['training_error'] = DummySeries()
179
180 series = {}
181
182 basedir = os.getcwd()
183
184 h5f = tables.openFile(os.path.join(basedir, nom_serie), "w")
185
186 #REDUCE_EVERY=10
187 # reconstruction
188 reconstruction_base = \
189 ErrorSeries(error_name="reconstruction_error",
190 table_name="reconstruction_error",
191 hdf5_file=h5f,
192 index_names=('epoch','minibatch'),
193 title="Reconstruction error (mean over "+str(REDUCE_EVERY)+" minibatches)")
194 series['reconstruction_error'] = \
195 AccumulatorSeriesWrapper(base_series=reconstruction_base,
196 reduce_every=REDUCE_EVERY)
197
198 # train
199 training_base = \
200 ErrorSeries(error_name="training_error",
201 table_name="training_error",
202 hdf5_file=h5f,
203 index_names=('epoch','minibatch'),
204 title="Training error (mean over "+str(REDUCE_EVERY)+" minibatches)")
205 series['training_error'] = \
206 AccumulatorSeriesWrapper(base_series=training_base,
207 reduce_every=REDUCE_EVERY)
208
209 # valid and test are not accumulated/mean, saved directly
210 series['validation_error'] = \
211 ErrorSeries(error_name="validation_error",
212 table_name="validation_error",
213 hdf5_file=h5f,
214 index_names=('epoch','minibatch'))
215
216 series['test_error'] = \
217 ErrorSeries(error_name="test_error",
218 table_name="test_error",
219 hdf5_file=h5f,
220 index_names=('epoch','minibatch'))
221
222 param_names = []
223 for i in range(num_hidden_layers):
224 param_names += ['layer%d_W'%i, 'layer%d_b'%i, 'layer%d_bprime'%i]
225 param_names += ['logreg_layer_W', 'logreg_layer_b']
226
227 # comment out series we don't want to save
228 series['params'] = SharedParamsStatisticsWrapper(
229 new_group_name="params",
230 base_group="/",
231 arrays_names=param_names,
232 hdf5_file=h5f,
233 index_names=('epoch',))
234
235 return series
236
237 # Perform insertion into the Postgre DB based on combination
238 # of hyperparameter values above
239 # (see comment for produit_cartesien_jobs() to know how it works)
240 def jobman_insert_nist():
241 jobs = produit_cartesien_jobs(JOB_VALS)
242
243 db = jobman.sql.db(JOBDB)
244 for job in jobs:
245 job.update({jobman.sql.EXPERIMENT: EXPERIMENT_PATH})
246 jobman.sql.insert_dict(job, db)
247
248 print "inserted"
249
250 if __name__ == '__main__':
251
252 args = sys.argv[1:]
253
254 #if len(args) > 0 and args[0] == 'load_nist':
255 # test_load_nist()
256
257 if len(args) > 0 and args[0] == 'jobman_insert':
258 jobman_insert_nist()
259
260 elif len(args) > 0 and args[0] == 'test_jobman_entrypoint':
261 chanmock = DD({'COMPLETE':0,'save':(lambda:None)})
262 jobman_entrypoint(DD(DEFAULT_HP_NIST), chanmock)
263
264 else:
265 print "Bad arguments"
266