Mercurial > ift6266

comparison deep/stacked_dae/v_sylvain/nist_sda.py @ 230:8a94a5c808cd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Repertoire pour faire les tests avec les differents ensembles pour le finetuning
author SylvainPL <sylvain.pannetier.lebeuf@umontreal.ca>
date Fri, 12 Mar 2010 16:47:10 -0500
parents
children c452e3a0a3b1
comparison
equal deleted inserted replaced
229:02eb98d051fe 230:8a94a5c808cd
1 #!/usr/bin/python
2 # coding: utf-8
3
4 import ift6266
5 import pylearn
6
7 import numpy
8 import theano
9 import time
10
11 import pylearn.version
12 import theano.tensor as T
13 from theano.tensor.shared_randomstreams import RandomStreams
14
15 import copy
16 import sys
17 import os
18 import os.path
19
20 from jobman import DD
21 import jobman, jobman.sql
22 from pylearn.io import filetensor
23
24 from ift6266 import datasets
25
26 from utils import produit_cartesien_jobs
27
28 from sgd_optimization import SdaSgdOptimizer
29
30 #from ift6266.utils.scalar_series import *
31 from ift6266.utils.seriestables import *
32 import tables
33
34 ##############################################################################
35 # GLOBALS
36
37 TEST_CONFIG = False
38
39 #NIST_ALL_LOCATION = '/data/lisa/data/nist/by_class/all'
40 JOBDB = 'postgres://ift6266h10@gershwin/ift6266h10_sandbox_db/fsavard_sda_v2'
41 EXPERIMENT_PATH = "ift6266.deep.stacked_dae.v2.nist_sda.jobman_entrypoint"
42
43 REDUCE_TRAIN_TO = None
44 MAX_FINETUNING_EPOCHS = 1000
45 # number of minibatches before taking means for valid error etc.
46 REDUCE_EVERY = 100
47
48 if TEST_CONFIG:
49 REDUCE_TRAIN_TO = 1000
50 MAX_FINETUNING_EPOCHS = 2
51 REDUCE_EVERY = 10
52
53 # Possible values the hyperparameters can take. These are then
54 # combined with produit_cartesien_jobs so we get a list of all
55 # possible combinations, each one resulting in a job inserted
56 # in the jobman DB.
57 JOB_VALS = {'pretraining_lr': [0.1, 0.01],#, 0.001],#, 0.0001],
58 'pretraining_epochs_per_layer': [10,20],
59 'hidden_layers_sizes': [300,800],
60 'corruption_levels': [0.1,0.2,0.3],
61 'minibatch_size': [20],
62 'max_finetuning_epochs':[MAX_FINETUNING_EPOCHS],
63 'finetuning_lr':[0.1, 0.01], #0.001 was very bad, so we leave it out
64 'num_hidden_layers':[2,3]}
65
66 # Just useful for tests... minimal number of epochs
67 DEFAULT_HP_NIST = DD({'finetuning_lr':0.1,
68 'pretraining_lr':0.1,
69 'pretraining_epochs_per_layer':2,
70 'max_finetuning_epochs':2,
71 'hidden_layers_sizes':800,
72 'corruption_levels':0.2,
73 'minibatch_size':20,
74 'reduce_train_to':10000,
75 'num_hidden_layers':1})
76
77 '''
78 Function called by jobman upon launching each job
79 Its path is the one given when inserting jobs: see EXPERIMENT_PATH
80 '''
81 def jobman_entrypoint(state, channel):
82 # record mercurial versions of each package
83 pylearn.version.record_versions(state,[theano,ift6266,pylearn])
84 # TODO: remove this, bad for number of simultaneous requests on DB
85 channel.save()
86
87 workingdir = os.getcwd()
88
89 ########### Il faudrait arranger ici pour train plus petit
90
91 ## print "Will load NIST"
92 ##
93 ## nist = NIST(minibatch_size=20)
94 ##
95 ## print "NIST loaded"
96 ##
97 ## # For test runs, we don't want to use the whole dataset so
98 ## # reduce it to fewer elements if asked to.
99 ## rtt = None
100 ## if state.has_key('reduce_train_to'):
101 ## rtt = state['reduce_train_to']
102 ## elif REDUCE_TRAIN_TO:
103 ## rtt = REDUCE_TRAIN_TO
104 ##
105 ## if rtt:
106 ## print "Reducing training set to "+str(rtt)+ " examples"
107 ## nist.reduce_train_set(rtt)
108 ##
109 ## train,valid,test = nist.get_tvt()
110 ## dataset = (train,valid,test)
111
112 n_ins = 32*32
113 n_outs = 62 # 10 digits, 26*2 (lower, capitals)
114
115 series = create_series(state.num_hidden_layers)
116
117 print "Creating optimizer with state, ", state
118
119 optimizer = SdaSgdOptimizer(dataset=datasets.nist_all, hyperparameters=state, \
120 n_ins=n_ins, n_outs=n_outs,\
121 series=series)
122
123 optimizer.pretrain()
124 channel.save()
125
126 optimizer.finetune()
127 channel.save()
128
129 return channel.COMPLETE
130
131 # These Series objects are used to save various statistics
132 # during the training.
133 def create_series(num_hidden_layers):
134
135 # Replace series we don't want to save with DummySeries, e.g.
136 # series['training_error'] = DummySeries()
137
138 series = {}
139
140 basedir = os.getcwd()
141
142 h5f = tables.openFile(os.path.join(basedir, "series.h5"), "w")
143
144 # reconstruction
145 reconstruction_base = \
146 ErrorSeries(error_name="reconstruction_error",
147 table_name="reconstruction_error",
148 hdf5_file=h5f,
149 index_names=('epoch','minibatch'),
150 title="Reconstruction error (mean over "+str(REDUCE_EVERY)+" minibatches)")
151 series['reconstruction_error'] = \
152 AccumulatorSeriesWrapper(base_series=reconstruction_base,
153 reduce_every=REDUCE_EVERY)
154
155 # train
156 training_base = \
157 ErrorSeries(error_name="training_error",
158 table_name="training_error",
159 hdf5_file=h5f,
160 index_names=('epoch','minibatch'),
161 title="Training error (mean over "+str(REDUCE_EVERY)+" minibatches)")
162 series['training_error'] = \
163 AccumulatorSeriesWrapper(base_series=training_base,
164 reduce_every=REDUCE_EVERY)
165
166 # valid and test are not accumulated/mean, saved directly
167 series['validation_error'] = \
168 ErrorSeries(error_name="validation_error",
169 table_name="validation_error",
170 hdf5_file=h5f,
171 index_names=('epoch','minibatch'))
172
173 series['test_error'] = \
174 ErrorSeries(error_name="test_error",
175 table_name="test_error",
176 hdf5_file=h5f,
177 index_names=('epoch','minibatch'))
178
179 param_names = []
180 for i in range(num_hidden_layers):
181 param_names += ['layer%d_W'%i, 'layer%d_b'%i, 'layer%d_bprime'%i]
182 param_names += ['logreg_layer_W', 'logreg_layer_b']
183
184 # comment out series we don't want to save
185 series['params'] = SharedParamsStatisticsWrapper(
186 new_group_name="params",
187 base_group="/",
188 arrays_names=param_names,
189 hdf5_file=h5f,
190 index_names=('epoch',))
191
192 return series
193
194 # Perform insertion into the Postgre DB based on combination
195 # of hyperparameter values above
196 # (see comment for produit_cartesien_jobs() to know how it works)
197 def jobman_insert_nist():
198 jobs = produit_cartesien_jobs(JOB_VALS)
199
200 db = jobman.sql.db(JOBDB)
201 for job in jobs:
202 job.update({jobman.sql.EXPERIMENT: EXPERIMENT_PATH})
203 jobman.sql.insert_dict(job, db)
204
205 print "inserted"
206
207 class NIST:
208 def __init__(self, minibatch_size, basepath=None, reduce_train_to=None):
209 global NIST_ALL_LOCATION
210
211 self.minibatch_size = minibatch_size
212 self.basepath = basepath and basepath or NIST_ALL_LOCATION
213
214 self.set_filenames()
215
216 # arrays of 2 elements: .x, .y
217 self.train = [None, None]
218 self.test = [None, None]
219
220 self.load_train_test()
221
222 self.valid = [[], []]
223 self.split_train_valid()
224 if reduce_train_to:
225 self.reduce_train_set(reduce_train_to)
226
227 def get_tvt(self):
228 return self.train, self.valid, self.test
229
230 def set_filenames(self):
231 self.train_files = ['all_train_data.ft',
232 'all_train_labels.ft']
233
234 self.test_files = ['all_test_data.ft',
235 'all_test_labels.ft']
236
237 def load_train_test(self):
238 self.load_data_labels(self.train_files, self.train)
239 self.load_data_labels(self.test_files, self.test)
240
241 def load_data_labels(self, filenames, pair):
242 for i, fn in enumerate(filenames):
243 f = open(os.path.join(self.basepath, fn))
244 pair[i] = filetensor.read(f)
245 f.close()
246
247 def reduce_train_set(self, max):
248 self.train[0] = self.train[0][:max]
249 self.train[1] = self.train[1][:max]
250
251 if max < len(self.test[0]):
252 for ar in (self.test, self.valid):
253 ar[0] = ar[0][:max]
254 ar[1] = ar[1][:max]
255
256 def split_train_valid(self):
257 test_len = len(self.test[0])
258
259 new_train_x = self.train[0][:-test_len]
260 new_train_y = self.train[1][:-test_len]
261
262 self.valid[0] = self.train[0][-test_len:]
263 self.valid[1] = self.train[1][-test_len:]
264
265 self.train[0] = new_train_x
266 self.train[1] = new_train_y
267
268 def test_load_nist():
269 print "Will load NIST"
270
271 import time
272 t1 = time.time()
273 nist = NIST(20)
274 t2 = time.time()
275
276 print "NIST loaded. time delta = ", t2-t1
277
278 tr,v,te = nist.get_tvt()
279
280 print "Lenghts: ", len(tr[0]), len(v[0]), len(te[0])
281
282 raw_input("Press any key")
283
284 if __name__ == '__main__':
285
286 import sys
287
288 args = sys.argv[1:]
289
290 if len(args) > 0 and args[0] == 'load_nist':
291 test_load_nist()
292
293 elif len(args) > 0 and args[0] == 'jobman_insert':
294 jobman_insert_nist()
295
296 elif len(args) > 0 and args[0] == 'test_jobman_entrypoint':
297 chanmock = DD({'COMPLETE':0,'save':(lambda:None)})
298 jobman_entrypoint(DEFAULT_HP_NIST, chanmock)
299
300 else:
301 print "Bad arguments"
302