Mercurial > pylearn

annotate pylearn/algorithms/tests/test_mcRBM.py @ 1000:d4a14c6c36e0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mcRBM - post code-review #1 with Guillaume
author James Bergstra <bergstrj@iro.umontreal.ca>
date Tue, 24 Aug 2010 19:24:54 -0400
parents
children 075c193afd1b
rev   line source
1000
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
1
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
2
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
3 from pylearn.algorithms.mcRBM import *
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
4
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
5
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
6 def test_reproduce_ranzato_hinton_2010(dataset='MAR', as_unittest=True):
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
7 dataset='MAR'
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
8 if dataset == 'MAR':
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
9 n_vis=105
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
10 n_patches=10240
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
11 else:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
12 R,C= 16,16 # the size of image patches
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
13 n_vis=R*C
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
14 n_patches=100000
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
15
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
16 n_train_iters=5000
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
17
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
18 n_burnin_steps=10000
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
19
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
20 l1_penalty=1e-3
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
21 no_l1_epochs = 10
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
22 effective_l1_penalty=0.0
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
23
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
24 epoch_size=n_patches
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
25 batchsize = 128
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
26 lr = 0.075 / batchsize
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
27 s_lr = TT.scalar()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
28 s_l1_penalty=TT.scalar()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
29 n_K=256
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
30 n_J=100
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
31
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
32 rbm = MeanCovRBM.new_from_dims(n_I=n_vis, n_K=n_K, n_J=n_J)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
33
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
34 smplr = sampler(rbm, n_particles=batchsize)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
35
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
36 def l2(X):
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
37 return numpy.sqrt((X**2).sum())
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
38 if dataset == 'MAR':
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
39 tile = pylearn.dataset_ops.image_patches.save_filters_of_ranzato_hinton_2010
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
40 else:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
41 def tile(X, fname):
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
42 _img = image_tiling.tile_raster_images(X,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
43 img_shape=(R,C),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
44 min_dynamic_range=1e-2)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
45 image_tiling.save_tiled_raster_images(_img, fname)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
46
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
47 batch_idx = TT.iscalar()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
48
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
49 if dataset == 'MAR':
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
50 train_batch = pylearn.dataset_ops.image_patches.ranzato_hinton_2010_op(batch_idx * batchsize + np.arange(batchsize))
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
51 else:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
52 train_batch = pylearn.dataset_ops.image_patches.image_patches(
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
53 s_idx = (batch_idx * batchsize + np.arange(batchsize)),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
54 dims = (n_patches,R,C),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
55 center=True,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
56 unitvar=True,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
57 dtype=floatX,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
58 rasterized=True)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
59
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
60 if not as_unittest:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
61 imgs_fn = function([batch_idx], outputs=train_batch)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
62
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
63 grads = contrastive_grad(
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
64 free_energy_fn=lambda v: free_energy_given_v(rbm, v),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
65 pos_v=train_batch,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
66 neg_v=smplr.positions[0],
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
67 params=list(rbm),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
68 other_cost=(l1(rbm.U)+l1(rbm.W)) * s_l1_penalty)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
69 sgd_ups = sgd_updates(
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
70 rbm.params,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
71 grads,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
72 stepsizes=[2*s_lr, .2*s_lr, .02*s_lr, .1*s_lr, .02*s_lr ])
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
73 learn_fn = function([batch_idx, s_lr, s_l1_penalty],
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
74 outputs=[
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
75 grads[0].norm(2),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
76 (sgd_ups[0][1] - sgd_ups[0][0]).norm(2),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
77 (sgd_ups[1][1] - sgd_ups[1][0]).norm(2),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
78 ],
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
79 updates = sgd_ups)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
80
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
81 print "Learning..."
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
82 normVF=1
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
83 last_epoch = -1
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
84 for jj in xrange(n_train_iters):
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
85 epoch = jj*batchsize / epoch_size
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
86
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
87 print_jj = epoch != last_epoch
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
88 last_epoch = epoch
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
89
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
90 if epoch > 10:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
91 break
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
92
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
93 if as_unittest and epoch == 5:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
94 U = rbm.U.value
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
95 W = rbm.W.value
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
96 def allclose(a,b):
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
97 return numpy.allclose(a,b,rtol=1.01,atol=1e-3)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
98 print ""
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
99 print "--------------"
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
100 print "assert allclose(l2(U), %f)"%l2(U)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
101 print "assert allclose(l2(W), %f)"%l2(W)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
102 print "assert allclose(U.min(), %f)"%U.min()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
103 print "assert allclose(U.max(), %f)"%U.max()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
104 print "assert allclose(W.min(),%f)"%W.min()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
105 print "assert allclose(W.max(), %f)"%W.max()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
106 print "--------------"
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
107
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
108 assert allclose(l2(U), 21.351664)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
109 assert allclose(l2(W), 6.275828)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
110 assert allclose(U.min(), -1.176703)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
111 assert allclose(U.max(), 0.859802)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
112 assert allclose(W.min(),-0.223128)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
113 assert allclose(W.max(), 0.227558 )
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
114
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
115 break
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
116
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
117 if print_jj:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
118 if not as_unittest:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
119 tile(imgs_fn(jj), "imgs_%06i.png"%jj)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
120 tile(smplr.positions[0].value, "sample_%06i.png"%jj)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
121 tile(rbm.U.value.T, "U_%06i.png"%jj)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
122 tile(rbm.W.value.T, "W_%06i.png"%jj)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
123
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
124 print 'saving samples', jj, 'epoch', jj/(epoch_size/batchsize)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
125
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
126 print 'l2(U)', l2(rbm.U.value),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
127 print 'l2(W)', l2(rbm.W.value)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
128
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
129 print 'U min max', rbm.U.value.min(), rbm.U.value.max(),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
130 print 'W min max', rbm.W.value.min(), rbm.W.value.max(),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
131 print 'a min max', rbm.a.value.min(), rbm.a.value.max(),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
132 print 'b min max', rbm.b.value.min(), rbm.b.value.max(),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
133 print 'c min max', rbm.c.value.min(), rbm.c.value.max()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
134
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
135 print 'parts min', smplr.positions[0].value.min(),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
136 print 'max',smplr.positions[0].value.max(),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
137 print 'HMC step', smplr.stepsize,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
138 print 'arate', smplr.avg_acceptance_rate
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
139
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
140 # Continue HMC chain
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
141 smplr.simulate()
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
142
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
143 # Do CD update
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
144 l2_of_Ugrad = learn_fn(jj,
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
145 lr/max(1, jj/(20*epoch_size/batchsize)),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
146 effective_l1_penalty)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
147
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
148 if print_jj:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
149 print 'l2(U_grad)', float(l2_of_Ugrad[0]),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
150 print 'l2(U_inc)', float(l2_of_Ugrad[1]),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
151 print 'l2(W_inc)', float(l2_of_Ugrad[2]),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
152 #print 'FE+', float(l2_of_Ugrad[2]),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
153 #print 'FE+[0]', float(l2_of_Ugrad[3]),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
154 #print 'FE+[1]', float(l2_of_Ugrad[4]),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
155 #print 'FE+[2]', float(l2_of_Ugrad[5]),
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
156 #print 'FE+[3]', float(l2_of_Ugrad[6])
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
157
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
158 if jj == no_l1_epochs * epoch_size/batchsize:
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
159 print "Activating L1 weight decay"
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
160 effective_l1_penalty = 1e-3
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
161
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
162 # weird normalization technique...
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
163 # It constrains all the columns of the matrix to have the same length
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
164 # But the matrix itself is re-scaled to have an arbitrary abslute size.
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
165 U = rbm.U.value
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
166 U_norms = np.sqrt((U*U).sum(axis=0))
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
167 assert len(U_norms) == n_K
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
168 normVF = .95 * normVF + .05 * np.mean(U_norms)
d4a14c6c36e0 mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff changeset
169 rbm.U.value = rbm.U.value * normVF/U_norms