Mercurial > ift6266

comparison deep/stacked_dae/v_guillaume/nist_sda.py @ 436:0ca069550abd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added : single class version of SDA
author Guillaume Sicard <guitch21@gmail.com>
date Mon, 03 May 2010 06:14:05 -0400
parents
children
comparison
equal deleted inserted replaced
435:d8129a09ffb1 436:0ca069550abd
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 # coding: utf-8
4
5 import ift6266
6 import pylearn
7
8 import numpy
9 import theano
10 import time
11
12 import pylearn.version
13 import theano.tensor as T
14 from theano.tensor.shared_randomstreams import RandomStreams
15
16 import copy
17 import sys
18 import os
19 import os.path
20
21 from jobman import DD
22 import jobman, jobman.sql
23 from pylearn.io import filetensor
24
25 from utils import produit_cartesien_jobs
26 from copy import copy
27
28 from sgd_optimization import SdaSgdOptimizer
29
30 #from ift6266.utils.scalar_series import *
31 from ift6266.utils.seriestables import *
32 import tables
33
34 from ift6266 import datasets
35 from config import *
36
37 '''
38 Function called by jobman upon launching each job
39 Its path is the one given when inserting jobs: see EXPERIMENT_PATH
40 '''
41 def jobman_entrypoint(state, channel):
42 # record mercurial versions of each package
43 pylearn.version.record_versions(state,[theano,ift6266,pylearn])
44 # TODO: remove this, bad for number of simultaneous requests on DB
45 channel.save()
46
47 # For test runs, we don't want to use the whole dataset so
48 # reduce it to fewer elements if asked to.
49 rtt = None
50 if state.has_key('reduce_train_to'):
51 rtt = state['reduce_train_to']
52 elif REDUCE_TRAIN_TO:
53 rtt = REDUCE_TRAIN_TO
54
55 if state.has_key('decrease_lr'):
56 decrease_lr = state['decrease_lr']
57 else :
58 decrease_lr = 0
59
60 if state.has_key('decrease_lr_pretrain'):
61 dec=state['decrease_lr_pretrain']
62 else :
63 dec=0
64
65 n_ins = 32*32
66
67 if state.has_key('subdataset'):
68 subdataset_name=state['subdataset']
69 else:
70 subdataset_name=SUBDATASET_NIST
71
72 #n_outs = 62 # 10 digits, 26*2 (lower, capitals)
73 if subdataset_name == "upper":
74 n_outs = 26
75 subdataset = datasets.nist_upper()
76 examples_per_epoch = NIST_UPPER_TRAIN_SIZE
77 elif subdataset_name == "lower":
78 n_outs = 26
79 subdataset = datasets.nist_lower()
80 examples_per_epoch = NIST_LOWER_TRAIN_SIZE
81 elif subdataset_name == "digits":
82 n_outs = 10
83 subdataset = datasets.nist_digits()
84 examples_per_epoch = NIST_DIGITS_TRAIN_SIZE
85 else:
86 n_outs = 62
87 subdataset = datasets.nist_all()
88 examples_per_epoch = NIST_ALL_TRAIN_SIZE
89
90 print 'Using subdataset ', subdataset_name
91
92 #To be sure variables will not be only in the if statement
93 PATH = ''
94 nom_reptrain = ''
95 nom_serie = ""
96 if state['pretrain_choice'] == 0:
97 nom_serie="series_NIST.h5"
98 elif state['pretrain_choice'] == 1:
99 nom_serie="series_P07.h5"
100
101 series = create_series(state.num_hidden_layers,nom_serie)
102
103
104 print "Creating optimizer with state, ", state
105
106 optimizer = SdaSgdOptimizer(dataset_name=subdataset_name,\
107 dataset=subdataset,\
108 hyperparameters=state, \
109 n_ins=n_ins, n_outs=n_outs,\
110 examples_per_epoch=examples_per_epoch, \
111 series=series,
112 max_minibatches=rtt)
113
114 parameters=[]
115 #Number of files of P07 used for pretraining
116 nb_file=0
117
118 print('\n\tpretraining with NIST\n')
119
120 optimizer.pretrain(subdataset, decrease = dec)
121
122 channel.save()
123
124 #Set some of the parameters used for the finetuning
125 if state.has_key('finetune_set'):
126 finetune_choice=state['finetune_set']
127 else:
128 finetune_choice=FINETUNE_SET
129
130 if state.has_key('max_finetuning_epochs'):
131 max_finetune_epoch_NIST=state['max_finetuning_epochs']
132 else:
133 max_finetune_epoch_NIST=MAX_FINETUNING_EPOCHS
134
135 if state.has_key('max_finetuning_epochs_P07'):
136 max_finetune_epoch_P07=state['max_finetuning_epochs_P07']
137 else:
138 max_finetune_epoch_P07=max_finetune_epoch_NIST
139
140 #Decide how the finetune is done
141
142 if finetune_choice == 0:
143 print('\n\n\tfinetune with NIST\n\n')
144 optimizer.reload_parameters('params_pretrain.txt')
145 optimizer.finetune(subdataset,subdataset,max_finetune_epoch_NIST,ind_test=1,decrease=decrease_lr)
146 channel.save()
147 if finetune_choice == 1:
148 print('\n\n\tfinetune with P07\n\n')
149 optimizer.reload_parameters('params_pretrain.txt')
150 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=0,decrease=decrease_lr)
151 channel.save()
152 if finetune_choice == 2:
153 print('\n\n\tfinetune with P07 followed by NIST\n\n')
154 optimizer.reload_parameters('params_pretrain.txt')
155 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=20,decrease=decrease_lr)
156 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=21,decrease=decrease_lr)
157 channel.save()
158 if finetune_choice == 3:
159 print('\n\n\tfinetune with NIST only on the logistic regression on top (but validation on P07).\n\
160 All hidden units output are input of the logistic regression\n\n')
161 optimizer.reload_parameters('params_pretrain.txt')
162 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,special=1,decrease=decrease_lr)
163
164
165 if finetune_choice==-1:
166 print('\nSERIE OF 4 DIFFERENT FINETUNINGS')
167 print('\n\n\tfinetune with NIST\n\n')
168 sys.stdout.flush()
169 optimizer.reload_parameters('params_pretrain.txt')
170 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,decrease=decrease_lr)
171 channel.save()
172 print('\n\n\tfinetune with P07\n\n')
173 sys.stdout.flush()
174 optimizer.reload_parameters('params_pretrain.txt')
175 optimizer.finetune(datasets.nist_P07(),datasets.nist_all(),max_finetune_epoch_P07,ind_test=0,decrease=decrease_lr)
176 channel.save()
177 print('\n\n\tfinetune with P07 (done earlier) followed by NIST (written here)\n\n')
178 sys.stdout.flush()
179 optimizer.reload_parameters('params_finetune_P07.txt')
180 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=21,decrease=decrease_lr)
181 channel.save()
182 print('\n\n\tfinetune with NIST only on the logistic regression on top.\n\
183 All hidden units output are input of the logistic regression\n\n')
184 sys.stdout.flush()
185 optimizer.reload_parameters('params_pretrain.txt')
186 optimizer.finetune(datasets.nist_all(),datasets.nist_P07(),max_finetune_epoch_NIST,ind_test=1,special=1,decrease=decrease_lr)
187 channel.save()
188
189 channel.save()
190
191 return channel.COMPLETE
192
193 # These Series objects are used to save various statistics
194 # during the training.
195 def create_series(num_hidden_layers, nom_serie):
196
197 # Replace series we don't want to save with DummySeries, e.g.
198 # series['training_error'] = DummySeries()
199
200 series = {}
201
202 basedir = os.getcwd()
203
204 h5f = tables.openFile(os.path.join(basedir, nom_serie), "w")
205
206 # reconstruction
207 reconstruction_base = \
208 ErrorSeries(error_name="reconstruction_error",
209 table_name="reconstruction_error",
210 hdf5_file=h5f,
211 index_names=('epoch','minibatch'),
212 title="Reconstruction error (mean over "+str(REDUCE_EVERY)+" minibatches)")
213 series['reconstruction_error'] = \
214 AccumulatorSeriesWrapper(base_series=reconstruction_base,
215 reduce_every=REDUCE_EVERY)
216
217 # train
218 training_base = \
219 ErrorSeries(error_name="training_error",
220 table_name="training_error",
221 hdf5_file=h5f,
222 index_names=('epoch','minibatch'),
223 title="Training error (mean over "+str(REDUCE_EVERY)+" minibatches)")
224 series['training_error'] = \
225 AccumulatorSeriesWrapper(base_series=training_base,
226 reduce_every=REDUCE_EVERY)
227
228 # valid and test are not accumulated/mean, saved directly
229 series['validation_error'] = \
230 ErrorSeries(error_name="validation_error",
231 table_name="validation_error",
232 hdf5_file=h5f,
233 index_names=('epoch','minibatch'))
234
235 series['test_error'] = \
236 ErrorSeries(error_name="test_error",
237 table_name="test_error",
238 hdf5_file=h5f,
239 index_names=('epoch','minibatch'))
240
241 param_names = []
242 for i in range(num_hidden_layers):
243 param_names += ['layer%d_W'%i, 'layer%d_b'%i, 'layer%d_bprime'%i]
244 param_names += ['logreg_layer_W', 'logreg_layer_b']
245
246 # comment out series we don't want to save
247 series['params'] = SharedParamsStatisticsWrapper(
248 new_group_name="params",
249 base_group="/",
250 arrays_names=param_names,
251 hdf5_file=h5f,
252 index_names=('epoch',))
253
254 return series
255
256 # Perform insertion into the Postgre DB based on combination
257 # of hyperparameter values above
258 # (see comment for produit_cartesien_jobs() to know how it works)
259 def jobman_insert_nist():
260 jobs = produit_cartesien_jobs(JOB_VALS)
261
262 db = jobman.sql.db(JOBDB)
263 for job in jobs:
264 job.update({jobman.sql.EXPERIMENT: EXPERIMENT_PATH})
265 jobman.sql.insert_dict(job, db)
266
267 print "inserted"
268
269 if __name__ == '__main__':
270
271 args = sys.argv[1:]
272
273 #if len(args) > 0 and args[0] == 'load_nist':
274 # test_load_nist()
275
276 if len(args) > 0 and args[0] == 'jobman_insert':
277 jobman_insert_nist()
278
279 elif len(args) > 0 and args[0] == 'test_jobman_entrypoint':
280 chanmock = DD({'COMPLETE':0,'save':(lambda:None)})
281 jobman_entrypoint(DD(DEFAULT_HP_NIST), chanmock)
282
283 else:
284 print "Bad arguments"
285