Mercurial > pylearn

comparison pylearn/algorithms/tests/test_mcRBM.py @ 1509:b709f6b53b17

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
auto fix white space.
author Frederic Bastien <nouiz@nouiz.org>
date Mon, 12 Sep 2011 11:46:27 -0400
parents b28e8730c948
children 7f166d01bf8e
comparison
equal deleted inserted replaced
1508:b28e8730c948 1509:b709f6b53b17
98 imgs_fn = theano.function([batch_idx], outputs=train_batch) 98 imgs_fn = theano.function([batch_idx], outputs=train_batch)
99 99
100 trainer = trainer_alloc( 100 trainer = trainer_alloc(
101 rbm_alloc(n_I=n_vis), 101 rbm_alloc(n_I=n_vis),
102 train_batch, 102 train_batch,
103 batchsize, 103 batchsize,
104 initial_lr_per_example=lr_per_example, 104 initial_lr_per_example=lr_per_example,
105 l1_penalty=l1_penalty, 105 l1_penalty=l1_penalty,
106 l1_penalty_start=l1_penalty_start, 106 l1_penalty_start=l1_penalty_start,
107 persistent_chains=persistent_chains) 107 persistent_chains=persistent_chains)
108 rbm=trainer.rbm 108 rbm=trainer.rbm
109 109
110 if persistent_chains: 110 if persistent_chains:
111 grads = trainer.contrastive_grads() 111 grads = trainer.contrastive_grads()
112 learn_fn = theano.function([batch_idx], 112 learn_fn = theano.function([batch_idx],
113 outputs=[grads[0].norm(2), grads[0].norm(2), grads[1].norm(2)], 113 outputs=[grads[0].norm(2), grads[0].norm(2), grads[1].norm(2)],
114 updates=trainer.cd_updates()) 114 updates=trainer.cd_updates())
115 else: 115 else:
116 learn_fn = theano.function([batch_idx], outputs=[], updates=trainer.cd_updates()) 116 learn_fn = theano.function([batch_idx], outputs=[], updates=trainer.cd_updates())
117 117
168 168
169 print 'saving samples', jj, 'epoch', jj/(epoch_size/batchsize) 169 print 'saving samples', jj, 'epoch', jj/(epoch_size/batchsize)
170 170
171 print 'l2(U)', l2(rbm.U.value), 171 print 'l2(U)', l2(rbm.U.value),
172 print 'l2(W)', l2(rbm.W.value), 172 print 'l2(W)', l2(rbm.W.value),
173 print 'l1_penalty', 173 print 'l1_penalty',
174 try: 174 try:
175 print trainer.effective_l1_penalty.value 175 print trainer.effective_l1_penalty.value
176 except: 176 except:
177 print trainer.effective_l1_penalty 177 print trainer.effective_l1_penalty
178 178
181 print 'a min max', rbm.a.value.min(), rbm.a.value.max(), 181 print 'a min max', rbm.a.value.min(), rbm.a.value.max(),
182 print 'b min max', rbm.b.value.min(), rbm.b.value.max(), 182 print 'b min max', rbm.b.value.min(), rbm.b.value.max(),
183 print 'c min max', rbm.c.value.min(), rbm.c.value.max() 183 print 'c min max', rbm.c.value.min(), rbm.c.value.max()
184 184
185 if persistent_chains: 185 if persistent_chains:
186 print 'parts min', smplr.positions.value.min(), 186 print 'parts min', smplr.positions.value.min(),
187 print 'max',smplr.positions.value.max(), 187 print 'max',smplr.positions.value.max(),
188 print 'HMC step', smplr.stepsize.value, 188 print 'HMC step', smplr.stepsize.value,
189 print 'arate', smplr.avg_acceptance_rate.value 189 print 'arate', smplr.avg_acceptance_rate.value
190 190
191 191
229 max_components=n_vis, seed=234) 229 max_components=n_vis, seed=234)
230 # 230 #
231 # Set up mcRBM Trainer 231 # Set up mcRBM Trainer
232 # Initialize P using topological 3x3 overlapping patches thing 232 # Initialize P using topological 3x3 overlapping patches thing
233 # start learning P matrix after 2 passes through dataset 233 # start learning P matrix after 2 passes through dataset
234 # 234 #
235 rbm_filename = 'mcRBM.rbm.%06i.pkl'%46000 235 rbm_filename = 'mcRBM.rbm.%06i.pkl'%46000
236 try: 236 try:
237 open(rbm_filename).close() 237 open(rbm_filename).close()
238 load_mcrbm = True 238 load_mcrbm = True
239 except: 239 except:
250 tile = pylearn.dataset_ops.tinyimages.save_filters 250 tile = pylearn.dataset_ops.tinyimages.save_filters
251 train_batch = theano.tensor.matrix() 251 train_batch = theano.tensor.matrix()
252 trainer = mcRBMTrainer.alloc_for_P( 252 trainer = mcRBMTrainer.alloc_for_P(
253 rbm=mcRBM_withP.alloc_topo_P(n_I=n_vis, n_J=81), 253 rbm=mcRBM_withP.alloc_topo_P(n_I=n_vis, n_J=81),
254 visible_batch=train_batch, 254 visible_batch=train_batch,
255 batchsize=batchsize, 255 batchsize=batchsize,
256 initial_lr_per_example=0.05, 256 initial_lr_per_example=0.05,
257 l1_penalty=1e-3, 257 l1_penalty=1e-3,
258 l1_penalty_start=sys.maxint, 258 l1_penalty_start=sys.maxint,
259 p_training_start=2*epoch_size//batchsize, 259 p_training_start=2*epoch_size//batchsize,
260 persistent_chains=False) 260 persistent_chains=False)
275 275
276 print 'saving samples', ii, 'epoch', i_epoch, i_batch 276 print 'saving samples', ii, 'epoch', i_epoch, i_batch
277 277
278 print 'l2(U)', l2(rbm.U.value), 278 print 'l2(U)', l2(rbm.U.value),
279 print 'l2(W)', l2(rbm.W.value), 279 print 'l2(W)', l2(rbm.W.value),
280 print 'l1_penalty', 280 print 'l1_penalty',
281 try: 281 try:
282 print trainer.effective_l1_penalty.value 282 print trainer.effective_l1_penalty.value
283 except: 283 except:
284 print trainer.effective_l1_penalty 284 print trainer.effective_l1_penalty
285 285
315 else: 315 else:
316 batchsize=100 316 batchsize=100
317 feat_idx = tensor.lscalar() 317 feat_idx = tensor.lscalar()
318 feat_idx_range = feat_idx * batchsize + tensor.arange(batchsize) 318 feat_idx_range = feat_idx * batchsize + tensor.arange(batchsize)
319 train_batch_x, train_batch_y = pylearn.dataset_ops.cifar10.cifar10( 319 train_batch_x, train_batch_y = pylearn.dataset_ops.cifar10.cifar10(
320 feat_idx_range, 320 feat_idx_range,
321 split='all', 321 split='all',
322 dtype='uint8', 322 dtype='uint8',
323 rasterized=False, 323 rasterized=False,
324 color='rgb') 324 color='rgb')
325 325
326 WINDOW_SIZE=8 326 WINDOW_SIZE=8
327 WINDOW_STRIDE=4 327 WINDOW_STRIDE=4
395 y_i = tensor.ivector() 395 y_i = tensor.ivector()
396 lr = tensor.scalar() 396 lr = tensor.scalar()
397 #l1_regularization = float(sys.argv[1]) #1.e-3 397 #l1_regularization = float(sys.argv[1]) #1.e-3
398 #l2_regularization = float(sys.argv[2]) #1.e-3*0 398 #l2_regularization = float(sys.argv[2]) #1.e-3*0
399 399
400 feature_logreg = LogisticRegression.new(x_i, 400 feature_logreg = LogisticRegression.new(x_i,
401 n_in = 11025, n_out=10, 401 n_in = 11025, n_out=10,
402 dtype=x_i.dtype) 402 dtype=x_i.dtype)
403 403
404 # marc'aurelio does this... 404 # marc'aurelio does this...
405 feature_logreg.w.value = numpy.random.RandomState(44).randn(11025,10)*.02 405 feature_logreg.w.value = numpy.random.RandomState(44).randn(11025,10)*.02
406 406
407 traincost = feature_logreg.nll(y_i).sum() 407 traincost = feature_logreg.nll(y_i).sum()
408 traincost = traincost + abs(feature_logreg.w).sum() * l1_regularization 408 traincost = traincost + abs(feature_logreg.w).sum() * l1_regularization
409 #traincost = traincost + (feature_logreg.w**2).sum() * l2_regularization 409 #traincost = traincost + (feature_logreg.w**2).sum() * l2_regularization
410 train_logreg_fn = theano.function([x_i, y_i, lr], 410 train_logreg_fn = theano.function([x_i, y_i, lr],
411 [feature_logreg.nll(y_i).mean(), 411 [feature_logreg.nll(y_i).mean(),
412 feature_logreg.errors(y_i).mean()], 412 feature_logreg.errors(y_i).mean()],
413 updates=pylearn.gd.sgd.sgd_updates( 413 updates=pylearn.gd.sgd.sgd_updates(
414 params=feature_logreg.params, 414 params=feature_logreg.params,
415 grads=tensor.grad(traincost, feature_logreg.params), 415 grads=tensor.grad(traincost, feature_logreg.params),
457 for i in xrange(10000/batchsize): 457 for i in xrange(10000/batchsize):
458 x_i = valid_features[i*batchsize:(i+1)*batchsize] 458 x_i = valid_features[i*batchsize:(i+1)*batchsize]
459 y_i = valid_labels[i*batchsize:(i+1)*batchsize] 459 y_i = valid_labels[i*batchsize:(i+1)*batchsize]
460 460
461 #lr=0.0 -> no learning, safe for validation set 461 #lr=0.0 -> no learning, safe for validation set
462 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0) 462 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0)
463 nlls.append(nll) 463 nlls.append(nll)
464 l01s.append(l01) 464 l01s.append(l01)
465 print 'validate log_reg', numpy.mean(nlls), numpy.mean(l01s) 465 print 'validate log_reg', numpy.mean(nlls), numpy.mean(l01s)
466 466
467 # test 467 # test
471 for i in xrange(len(test_features)//batchsize): 471 for i in xrange(len(test_features)//batchsize):
472 x_i = test_features[i*batchsize:(i+1)*batchsize] 472 x_i = test_features[i*batchsize:(i+1)*batchsize]
473 y_i = test_labels[i*batchsize:(i+1)*batchsize] 473 y_i = test_labels[i*batchsize:(i+1)*batchsize]
474 474
475 #lr=0.0 -> no learning, safe for validation set 475 #lr=0.0 -> no learning, safe for validation set
476 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0) 476 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0)
477 nlls.append(nll) 477 nlls.append(nll)
478 l01s.append(l01) 478 l01s.append(l01)
479 print 'test log_reg', numpy.mean(nlls), numpy.mean(l01s) 479 print 'test log_reg', numpy.mean(nlls), numpy.mean(l01s)
480 480
481 #train 481 #train
493 493
494 494
495 import pickle as cPickle 495 import pickle as cPickle
496 #import cPickle 496 #import cPickle
497 if __name__ == '__main__': 497 if __name__ == '__main__':
498 if 0: 498 if 0:
499 #learning 16 x 16 pinwheel filters from official cifar patches (MAR) 499 #learning 16 x 16 pinwheel filters from official cifar patches (MAR)
500 rbm,smplr = test_reproduce_ranzato_hinton_2010( 500 rbm,smplr = test_reproduce_ranzato_hinton_2010(
501 as_unittest=False, 501 as_unittest=False,
502 n_train_iters=5000, 502 n_train_iters=5000,
503 rbm_alloc=lambda n_I : mcRBM_withP.alloc_topo_P(n_I, n_J=81), 503 rbm_alloc=lambda n_I : mcRBM_withP.alloc_topo_P(n_I, n_J=81),
522 522
523 if 1: 523 if 1:
524 def checkpoint(): 524 def checkpoint():
525 return checkpoint 525 return checkpoint
526 run_classif_experiment(checkpoint=checkpoint) 526 run_classif_experiment(checkpoint=checkpoint)
527