Mercurial > ift6266

comparison deep/convolutional_dae/scdae.py @ 288:80ee63c3e749

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add net saving (only the best model) and error saving using SeriesTable
author Arnaud Bergeron <abergeron@gmail.com>
date Fri, 26 Mar 2010 17:24:17 -0400
parents 20ebc1f2a9fe
children 518589bfee55
comparison
equal deleted inserted replaced
286:1cc535f3e254 288:80ee63c3e749
5 import numpy 5 import numpy
6 import theano 6 import theano
7 import theano.tensor as T 7 import theano.tensor as T
8 8
9 from itertools import izip 9 from itertools import izip
10 from ift6266.utils.seriestables import *
10 11
11 class cdae(LayerStack): 12 class cdae(LayerStack):
12 def __init__(self, filter_size, num_filt, num_in, subsampling, corruption, 13 def __init__(self, filter_size, num_filt, num_in, subsampling, corruption,
13 dtype, img_shape): 14 dtype, img_shape):
14 LayerStack.__init__(self, [ConvAutoencoder(filter_size=filter_size, 15 LayerStack.__init__(self, [ConvAutoencoder(filter_size=filter_size,
66 def build_funcs(batch_size, img_size, filter_sizes, num_filters, subs, 67 def build_funcs(batch_size, img_size, filter_sizes, num_filters, subs,
67 noise, mlp_sizes, out_size, dtype, pretrain_lr, train_lr): 68 noise, mlp_sizes, out_size, dtype, pretrain_lr, train_lr):
68 69
69 n = scdae_net((1,)+img_size, batch_size, filter_sizes, num_filters, subs, 70 n = scdae_net((1,)+img_size, batch_size, filter_sizes, num_filters, subs,
70 noise, mlp_sizes, out_size, dtype, batch_size) 71 noise, mlp_sizes, out_size, dtype, batch_size)
72
73 n.save('start.net')
74
71 x = T.fmatrix('x') 75 x = T.fmatrix('x')
72 y = T.ivector('y') 76 y = T.ivector('y')
73 77
74 def pretrainfunc(net, alpha): 78 def pretrainfunc(net, alpha):
75 up = trainers.get_updates(net.params, net.cost, alpha) 79 up = trainers.get_updates(net.params, net.cost, alpha)
108 return f2(x, y) 112 return f2(x, y)
109 return f 113 return f
110 114
111 trainf = select_f2(trainf_opt, trainf_reg, batch_size) 115 trainf = select_f2(trainf_opt, trainf_reg, batch_size)
112 evalf = select_f2(evalf_opt, evalf_reg, batch_size) 116 evalf = select_f2(evalf_opt, evalf_reg, batch_size)
113 return pretrain_funcs, trainf, evalf 117 return pretrain_funcs, trainf, evalf, n
114 118
115 def do_pretrain(pretrain_funcs, pretrain_epochs): 119 def do_pretrain(pretrain_funcs, pretrain_epochs, serie):
116 for f in pretrain_funcs: 120 for layer, f in enumerate(pretrain_funcs):
117 for i in xrange(pretrain_epochs): 121 for epoch in xrange(pretrain_epochs):
118 f() 122 serie.append((layer, epoch), f())
119 123
120 def massage_funcs(train_it, dset, batch_size, pretrain_funcs, trainf, evalf): 124 def massage_funcs(train_it, dset, batch_size, pretrain_funcs, trainf, evalf):
121 def pretrain_f(f): 125 def pretrain_f(f):
122 def res(): 126 def res():
123 for x, y in train_it: 127 for x, y in train_it:
154 def repeat_itf(itf, *args, **kwargs): 158 def repeat_itf(itf, *args, **kwargs):
155 while True: 159 while True:
156 for e in itf(*args, **kwargs): 160 for e in itf(*args, **kwargs):
157 yield e 161 yield e
158 162
163 def create_series():
164 import tables
165
166 series = {}
167 h5f = tables.openFile('series.h5', 'w')
168
169 series['recons_error'] = AccumulatorSeriesWrapper(
170 base_series=ErrorSeries(error_name='reconstruction_error',
171 table_name='reconstruction_error',
172 hdf5_file=h5f,
173 index_names=('layer', 'epoch'),
174 title="Reconstruction error (mse)")
175 reduce_every=100)
176
177 series['training_err'] = AccumulatorSeriesWrapper(
178 base_series=ErrorSeries(error_name='training_error',
179 table_name='training_error'
180 hdf5_file=h5f,
181 index_names=('iter',),
182 titles='Training error (nll)')
183 reduce_every=100)
184
185 series['valid_err'] = ErrorSeries(error_name='valid_error',
186 table_name='valid_error'
187 hdf5_file=h5f,
188 index_names=('iter',),
189 titles='Validation error (class)')
190
191 series['test_err'] = ErrorSeries(error_name='test_error',
192 table_name='test_error'
193 hdf5_file=h5f,
194 index_names=('iter',),
195 titles='Test error (class)')
196
159 def run_exp(state, channel): 197 def run_exp(state, channel):
160 from ift6266 import datasets 198 from ift6266 import datasets
161 from sgd_opt import sgd_opt 199 from sgd_opt import sgd_opt
162 import sys, time 200 import sys, time
163 201
164 channel.save()
165
166 # params: bsize, pretrain_lr, train_lr, nfilts1, nfilts2, nftils3, nfilts4 202 # params: bsize, pretrain_lr, train_lr, nfilts1, nfilts2, nftils3, nfilts4
167 # pretrain_rounds 203 # pretrain_rounds
204
205 pylearn.version.record_versions(state, [theano,ift6266,pylearn])
206 # TODO: maybe record pynnet version?
207 channel.save()
168 208
169 dset = dataset.nist_all() 209 dset = dataset.nist_all()
170 210
171 nfilts = [] 211 nfilts = []
172 if state.nfilts1 != 0: 212 if state.nfilts1 != 0:
180 220
181 fsizes = [(5,5)]*len(nfilts) 221 fsizes = [(5,5)]*len(nfilts)
182 subs = [(2,2)]*len(nfilts) 222 subs = [(2,2)]*len(nfilts)
183 noise = [state.noise]*len(nfilts) 223 noise = [state.noise]*len(nfilts)
184 224
185 pretrain_funcs, trainf, evalf = build_funcs( 225 pretrain_funcs, trainf, evalf, net = build_funcs(
186 img_size=(32, 32), 226 img_size=(32, 32),
187 batch_size=state.bsize, 227 batch_size=state.bsize,
188 filter_sizes=fsizes, 228 filter_sizes=fsizes,
189 num_filters=nfilts, 229 num_filters=nfilts,
190 subs=subs, 230 subs=subs,
196 train_lr=state.train_lr) 236 train_lr=state.train_lr)
197 237
198 pretrain_fs, train, valid, test = massage_funcs( 238 pretrain_fs, train, valid, test = massage_funcs(
199 state.bsize, dset, pretrain_funcs, trainf, evalf) 239 state.bsize, dset, pretrain_funcs, trainf, evalf)
200 240
201 do_pretrain(pretrain_fs, state.pretrain_rounds) 241 series = create_series()
242
243 do_pretrain(pretrain_fs, state.pretrain_rounds, series['recons_error'])
202 244
203 sgd_opt(train, valid, test, training_epochs=100000, patience=10000, 245 sgd_opt(train, valid, test, training_epochs=100000, patience=10000,
204 patience_increase=2., improvement_threshold=0.995, 246 patience_increase=2., improvement_threshold=0.995,
205 validation_frequency=2500) 247 validation_frequency=2500, series=series, net=net)
206 248
207 if __name__ == '__main__': 249 if __name__ == '__main__':
208 from ift6266 import datasets 250 from ift6266 import datasets
209 from sgd_opt import sgd_opt 251 from sgd_opt import sgd_opt
210 import sys, time 252 import sys, time
211 253
212 batch_size = 100 254 batch_size = 100
213 dset = datasets.mnist() 255 dset = datasets.mnist()
214 256
215 pretrain_funcs, trainf, evalf = build_funcs( 257 pretrain_funcs, trainf, evalf, net = build_funcs(
216 img_size = (28, 28), 258 img_size = (28, 28),
217 batch_size=batch_size, filter_sizes=[(5,5), (3,3)], 259 batch_size=batch_size, filter_sizes=[(5,5), (3,3)],
218 num_filters=[4, 4], subs=[(2,2), (2,2)], noise=[0.2, 0.2], 260 num_filters=[4, 4], subs=[(2,2), (2,2)], noise=[0.2, 0.2],
219 mlp_sizes=[500], out_size=10, dtype=numpy.float32, 261 mlp_sizes=[500], out_size=10, dtype=numpy.float32,
220 pretrain_lr=0.01, train_lr=0.1) 262 pretrain_lr=0.01, train_lr=0.1)
225 pretrain_funcs, trainf, evalf) 267 pretrain_funcs, trainf, evalf)
226 268
227 print "pretraining ...", 269 print "pretraining ...",
228 sys.stdout.flush() 270 sys.stdout.flush()
229 start = time.time() 271 start = time.time()
230 do_pretrain(pretrain_fs, 2500) 272 do_pretrain(pretrain_fs, 2500, DummySeries())
231 end = time.time() 273 end = time.time()
232 print "done (in", end-start, "s)" 274 print "done (in", end-start, "s)"
233 275
234 sgd_opt(train, valid, test, training_epochs=10000, patience=1000, 276 sgd_opt(train, valid, test, training_epochs=10000, patience=1000,
235 patience_increase=2., improvement_threshold=0.995, 277 patience_increase=2., improvement_threshold=0.995,