Mercurial > ift6266

comparison deep/stacked_dae/old/nist_sda.py @ 265:c8fe09a65039

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Déplacer le nouveau code de stacked_dae de v2 vers le répertoire de base 'stacked_dae', et bougé le vieux code vers le répertoire 'old'
author fsavard
date Fri, 19 Mar 2010 10:54:39 -0400
parents deep/stacked_dae/nist_sda.py@acb942530923
children
comparison
equal deleted inserted replaced
243:3c54cb3713ef 265:c8fe09a65039
1 #!/usr/bin/python
2 # coding: utf-8
3
4 import ift6266
5 import pylearn
6
7 import numpy
8 import theano
9 import time
10
11 import pylearn.version
12 import theano.tensor as T
13 from theano.tensor.shared_randomstreams import RandomStreams
14
15 import copy
16 import sys
17 import os
18 import os.path
19
20 from jobman import DD
21 import jobman, jobman.sql
22 from pylearn.io import filetensor
23
24 from utils import produit_cartesien_jobs
25
26 from sgd_optimization import SdaSgdOptimizer
27
28 from ift6266.utils.scalar_series import *
29
30 ##############################################################################
31 # GLOBALS
32
33 TEST_CONFIG = False
34
35 NIST_ALL_LOCATION = '/data/lisa/data/nist/by_class/all'
36 JOBDB = 'postgres://ift6266h10@gershwin/ift6266h10_db/fsavard_sda4'
37 EXPERIMENT_PATH = "ift6266.deep.stacked_dae.nist_sda.jobman_entrypoint"
38
39 REDUCE_TRAIN_TO = None
40 MAX_FINETUNING_EPOCHS = 1000
41 # number of minibatches before taking means for valid error etc.
42 REDUCE_EVERY = 1000
43
44 if TEST_CONFIG:
45 REDUCE_TRAIN_TO = 1000
46 MAX_FINETUNING_EPOCHS = 2
47 REDUCE_EVERY = 10
48
49 # Possible values the hyperparameters can take. These are then
50 # combined with produit_cartesien_jobs so we get a list of all
51 # possible combinations, each one resulting in a job inserted
52 # in the jobman DB.
53 JOB_VALS = {'pretraining_lr': [0.1, 0.01],#, 0.001],#, 0.0001],
54 'pretraining_epochs_per_layer': [10,20],
55 'hidden_layers_sizes': [300,800],
56 'corruption_levels': [0.1,0.2,0.3],
57 'minibatch_size': [20],
58 'max_finetuning_epochs':[MAX_FINETUNING_EPOCHS],
59 'finetuning_lr':[0.1, 0.01], #0.001 was very bad, so we leave it out
60 'num_hidden_layers':[2,3]}
61
62 # Just useful for tests... minimal number of epochs
63 DEFAULT_HP_NIST = DD({'finetuning_lr':0.1,
64 'pretraining_lr':0.1,
65 'pretraining_epochs_per_layer':20,
66 'max_finetuning_epochs':2,
67 'hidden_layers_sizes':800,
68 'corruption_levels':0.2,
69 'minibatch_size':20,
70 #'reduce_train_to':300,
71 'num_hidden_layers':2})
72
73 '''
74 Function called by jobman upon launching each job
75 Its path is the one given when inserting jobs:
76 ift6266.deep.stacked_dae.nist_sda.jobman_entrypoint
77 '''
78 def jobman_entrypoint(state, channel):
79 # record mercurial versions of each package
80 pylearn.version.record_versions(state,[theano,ift6266,pylearn])
81 channel.save()
82
83 workingdir = os.getcwd()
84
85 print "Will load NIST"
86
87 nist = NIST(minibatch_size=20)
88
89 print "NIST loaded"
90
91 # For test runs, we don't want to use the whole dataset so
92 # reduce it to fewer elements if asked to.
93 rtt = None
94 if state.has_key('reduce_train_to'):
95 rtt = state['reduce_train_to']
96 elif REDUCE_TRAIN_TO:
97 rtt = REDUCE_TRAIN_TO
98
99 if rtt:
100 print "Reducing training set to "+str(rtt)+ " examples"
101 nist.reduce_train_set(rtt)
102
103 train,valid,test = nist.get_tvt()
104 dataset = (train,valid,test)
105
106 n_ins = 32*32
107 n_outs = 62 # 10 digits, 26*2 (lower, capitals)
108
109 # b,b',W for each hidden layer
110 # + b,W of last layer (logreg)
111 numparams = state.num_hidden_layers * 3 + 2
112 series_mux = None
113 series_mux = create_series(workingdir, numparams)
114
115 print "Creating optimizer with state, ", state
116
117 optimizer = SdaSgdOptimizer(dataset=dataset, hyperparameters=state, \
118 n_ins=n_ins, n_outs=n_outs,\
119 input_divider=255.0, series_mux=series_mux)
120
121 optimizer.pretrain()
122 channel.save()
123
124 optimizer.finetune()
125 channel.save()
126
127 return channel.COMPLETE
128
129 # These Series objects are used to save various statistics
130 # during the training.
131 def create_series(basedir, numparams):
132 mux = SeriesMultiplexer()
133
134 # comment out series we don't want to save
135 mux.add_series(AccumulatorSeries(name="reconstruction_error",
136 reduce_every=REDUCE_EVERY, # every 1000 batches, we take the mean and save
137 mean=True,
138 directory=basedir, flush_every=1))
139
140 mux.add_series(AccumulatorSeries(name="training_error",
141 reduce_every=REDUCE_EVERY, # every 1000 batches, we take the mean and save
142 mean=True,
143 directory=basedir, flush_every=1))
144
145 mux.add_series(BaseSeries(name="validation_error", directory=basedir, flush_every=1))
146 mux.add_series(BaseSeries(name="test_error", directory=basedir, flush_every=1))
147
148 mux.add_series(ParamsArrayStats(numparams,name="params",directory=basedir))
149
150 return mux
151
152 # Perform insertion into the Postgre DB based on combination
153 # of hyperparameter values above
154 # (see comment for produit_cartesien_jobs() to know how it works)
155 def jobman_insert_nist():
156 jobs = produit_cartesien_jobs(JOB_VALS)
157
158 db = jobman.sql.db(JOBDB)
159 for job in jobs:
160 job.update({jobman.sql.EXPERIMENT: EXPERIMENT_PATH})
161 jobman.sql.insert_dict(job, db)
162
163 print "inserted"
164
165 class NIST:
166 def __init__(self, minibatch_size, basepath=None, reduce_train_to=None):
167 global NIST_ALL_LOCATION
168
169 self.minibatch_size = minibatch_size
170 self.basepath = basepath and basepath or NIST_ALL_LOCATION
171
172 self.set_filenames()
173
174 # arrays of 2 elements: .x, .y
175 self.train = [None, None]
176 self.test = [None, None]
177
178 self.load_train_test()
179
180 self.valid = [[], []]
181 self.split_train_valid()
182 if reduce_train_to:
183 self.reduce_train_set(reduce_train_to)
184
185 def get_tvt(self):
186 return self.train, self.valid, self.test
187
188 def set_filenames(self):
189 self.train_files = ['all_train_data.ft',
190 'all_train_labels.ft']
191
192 self.test_files = ['all_test_data.ft',
193 'all_test_labels.ft']
194
195 def load_train_test(self):
196 self.load_data_labels(self.train_files, self.train)
197 self.load_data_labels(self.test_files, self.test)
198
199 def load_data_labels(self, filenames, pair):
200 for i, fn in enumerate(filenames):
201 f = open(os.path.join(self.basepath, fn))
202 pair[i] = filetensor.read(f)
203 f.close()
204
205 def reduce_train_set(self, max):
206 self.train[0] = self.train[0][:max]
207 self.train[1] = self.train[1][:max]
208
209 if max < len(self.test[0]):
210 for ar in (self.test, self.valid):
211 ar[0] = ar[0][:max]
212 ar[1] = ar[1][:max]
213
214 def split_train_valid(self):
215 test_len = len(self.test[0])
216
217 new_train_x = self.train[0][:-test_len]
218 new_train_y = self.train[1][:-test_len]
219
220 self.valid[0] = self.train[0][-test_len:]
221 self.valid[1] = self.train[1][-test_len:]
222
223 self.train[0] = new_train_x
224 self.train[1] = new_train_y
225
226 def test_load_nist():
227 print "Will load NIST"
228
229 import time
230 t1 = time.time()
231 nist = NIST(20)
232 t2 = time.time()
233
234 print "NIST loaded. time delta = ", t2-t1
235
236 tr,v,te = nist.get_tvt()
237
238 print "Lenghts: ", len(tr[0]), len(v[0]), len(te[0])
239
240 raw_input("Press any key")
241
242 if __name__ == '__main__':
243
244 import sys
245
246 args = sys.argv[1:]
247
248 if len(args) > 0 and args[0] == 'load_nist':
249 test_load_nist()
250
251 elif len(args) > 0 and args[0] == 'jobman_insert':
252 jobman_insert_nist()
253
254 elif len(args) > 0 and args[0] == 'test_jobman_entrypoint':
255 chanmock = DD({'COMPLETE':0,'save':(lambda:None)})
256 jobman_entrypoint(DEFAULT_HP_NIST, chanmock)
257
258 else:
259 print "Bad arguments"
260