Mercurial > pylearn
annotate pylearn/algorithms/tests/test_mcRBM.py @ 1524:9d21919e2332
autopep8
author | Frederic Bastien <nouiz@nouiz.org> |
---|---|
date | Fri, 02 Nov 2012 13:02:18 -0400 |
parents | 7f166d01bf8e |
children | 9c24a2bdbe90 |
rev | line source |
---|---|
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
1 import sys |
1507 | 2 |
3 import numpy | |
4 import theano | |
5 from theano import tensor | |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
6 |
1507 | 7 from pylearn.algorithms.mcRBM import mcRBM, mcRBMTrainer, mcRBM_withP, l2 |
8 #import pylearn.datasets.cifar10 | |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
9 import pylearn.dataset_ops.cifar10 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
10 from pylearn.shared.layers.logreg import LogisticRegression |
1507 | 11 from pylearn.io import image_tiling |
12 import pylearn.dataset_ops.image_patches | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
13 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
14 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
15 def _default_rbm_alloc(n_I, n_K=256, n_J=100): |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
16 return mcRBM.alloc(n_I, n_K, n_J) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
17 |
1524 | 18 |
1508 | 19 def _default_trainer_alloc(rbm, train_batch, batchsize, initial_lr_per_example, |
20 l1_penalty, l1_penalty_start, persistent_chains): | |
1524 | 21 return mcRBMTrainer.alloc(rbm, train_batch, batchsize, |
22 l1_penalty=l1_penalty, | |
23 l1_penalty_start=l1_penalty_start, | |
24 persistent_chains=persistent_chains) | |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
25 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
26 |
1524 | 27 def test_reproduce_ranzato_hinton_2010( |
28 dataset='MAR', as_unittest=True, n_train_iters=5000, | |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
29 rbm_alloc=_default_rbm_alloc, trainer_alloc=_default_trainer_alloc, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
30 lr_per_example=.075, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
31 l1_penalty=1e-3, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
32 l1_penalty_start=1000, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
33 persistent_chains=True, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
34 ): |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
35 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
36 batchsize = 128 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
37 |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
38 if dataset == 'MAR': |
1524 | 39 n_vis = 105 |
40 n_patches = 10240 | |
41 epoch_size = n_patches | |
42 elif dataset == 'cifar10patches8x8': | |
43 R, C = 8, 8 # the size of image patches | |
44 n_vis = 96 # pca components | |
45 epoch_size = batchsize * 500 | |
46 n_patches = epoch_size * 20 | |
47 elif dataset == 'tinyimages_patches': | |
48 R, C = 8, 8 | |
49 n_vis = 81 | |
50 epoch_size = batchsize * 500 | |
51 n_patches = epoch_size * 20 | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
52 else: |
1524 | 53 R, C = 16, 16 # the size of image patches |
54 n_vis = R * C | |
55 n_patches = 100000 | |
56 epoch_size = n_patches | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
57 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
58 def l2(X): |
1524 | 59 return numpy.sqrt((X ** 2).sum()) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
60 |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
61 if dataset == 'MAR': |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
62 tile = pylearn.dataset_ops.image_patches.save_filters_of_ranzato_hinton_2010 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
63 elif dataset == 'cifar10patches8x8': |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
64 def tile(X, fname): |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
65 _img = pylearn.datasets.cifar10.tile_rasterized_examples( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
66 pylearn.preprocessing.pca.pca_whiten_inverse( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
67 pylearn.dataset_ops.cifar10.random_cifar_patches_pca( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
68 n_vis, None, 'float32', n_patches, R, C,), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
69 X), |
1524 | 70 img_shape=(R, C)) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
71 image_tiling.save_tiled_raster_images(_img, fname) |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
72 elif dataset == 'tinyimages_patches': |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
73 tile = pylearn.dataset_ops.tinyimages.save_filters |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
74 else: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
75 def tile(X, fname): |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
76 _img = image_tiling.tile_raster_images(X, |
1524 | 77 img_shape=(R, C), |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
78 min_dynamic_range=1e-2) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
79 image_tiling.save_tiled_raster_images(_img, fname) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
80 |
1507 | 81 batch_idx = tensor.iscalar() |
1524 | 82 batch_range = batch_idx * batchsize + numpy.arange(batchsize) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
83 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
84 if dataset == 'MAR': |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
85 train_batch = pylearn.dataset_ops.image_patches.ranzato_hinton_2010_op(batch_range) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
86 elif dataset == 'cifar10patches8x8': |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
87 train_batch = pylearn.dataset_ops.cifar10.cifar10_patches( |
1524 | 88 batch_range, 'train', n_patches=n_patches, patch_size=(R, C), |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
89 pca_components=n_vis) |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
90 elif dataset == 'tinyimages_patches': |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
91 train_batch = pylearn.dataset_ops.tinyimages.tinydataset_op(batch_range) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
92 else: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
93 train_batch = pylearn.dataset_ops.image_patches.image_patches( |
1524 | 94 s_idx=(batch_idx * batchsize + numpy.arange(batchsize)), |
95 dims=(n_patches, R, C), | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
96 center=True, |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
97 unitvar=True, |
1507 | 98 dtype=theano.config.floatX, |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
99 rasterized=True) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
100 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
101 if not as_unittest: |
1507 | 102 imgs_fn = theano.function([batch_idx], outputs=train_batch) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
103 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
104 trainer = trainer_alloc( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
105 rbm_alloc(n_I=n_vis), |
1267
075c193afd1b
refactoring mcRBM
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1000
diff
changeset
|
106 train_batch, |
1509 | 107 batchsize, |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
108 initial_lr_per_example=lr_per_example, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
109 l1_penalty=l1_penalty, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
110 l1_penalty_start=l1_penalty_start, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
111 persistent_chains=persistent_chains) |
1524 | 112 rbm = trainer.rbm |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
113 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
114 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
115 grads = trainer.contrastive_grads() |
1509 | 116 learn_fn = theano.function([batch_idx], |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
117 outputs=[grads[0].norm(2), grads[0].norm(2), grads[1].norm(2)], |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
118 updates=trainer.cd_updates()) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
119 else: |
1524 | 120 learn_fn = theano.function([batch_idx], outputs=[], |
121 updates=trainer.cd_updates()) | |
1267
075c193afd1b
refactoring mcRBM
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1000
diff
changeset
|
122 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
123 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
124 smplr = trainer.sampler |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
125 else: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
126 smplr = trainer._last_cd1_sampler |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
127 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
128 if dataset == 'cifar10patches8x8': |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
129 cPickle.dump( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
130 pylearn.dataset_ops.cifar10.random_cifar_patches_pca( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
131 n_vis, None, 'float32', n_patches, R, C,), |
1524 | 132 open('test_mcRBM.pca.pkl', 'w')) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
133 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
134 print "Learning..." |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
135 last_epoch = -1 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
136 for jj in xrange(n_train_iters): |
1524 | 137 epoch = jj * batchsize / epoch_size |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
138 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
139 print_jj = epoch != last_epoch |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
140 last_epoch = epoch |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
141 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
142 if as_unittest and epoch == 5: |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
143 U = rbm.U.get_value(borrow=True) |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
144 W = rbm.W.get_value(borrow=True) |
1524 | 145 |
146 def allclose(a, b): | |
147 return numpy.allclose(a, b, rtol=1.01, atol=1e-3) | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
148 print "" |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
149 print "--------------" |
1524 | 150 print "assert allclose(l2(U), %f)" % l2(U) |
151 print "assert allclose(l2(W), %f)" % l2(W) | |
152 print "assert allclose(U.min(), %f)" % U.min() | |
153 print "assert allclose(U.max(), %f)" % U.max() | |
154 print "assert allclose(W.min(),%f)" % W.min() | |
155 print "assert allclose(W.max(), %f)" % W.max() | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
156 print "--------------" |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
157 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
158 assert allclose(l2(U), 21.351664) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
159 assert allclose(l2(W), 6.275828) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
160 assert allclose(U.min(), -1.176703) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
161 assert allclose(U.max(), 0.859802) |
1524 | 162 assert allclose(W.min(), -0.223128) |
163 assert allclose(W.max(), 0.227558) | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
164 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
165 break |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
166 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
167 if print_jj: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
168 if not as_unittest: |
1524 | 169 tile(imgs_fn(jj), "imgs_%06i.png" % jj) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
170 if persistent_chains: |
1524 | 171 tile(smplr.positions.value, "sample_%06i.png" % jj) |
172 tile(rbm.U.value.T, "U_%06i.png" % jj) | |
173 tile(rbm.W.value.T, "W_%06i.png" % jj) | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
174 |
1524 | 175 print 'saving samples', jj, 'epoch', jj / (epoch_size / batchsize) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
176 |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
177 print 'l2(U)', l2(rbm.U.get_value(borrow=True)), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
178 print 'l2(W)', l2(rbm.W.get_value(borrow=True)), |
1509 | 179 print 'l1_penalty', |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
180 try: |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
181 print trainer.effective_l1_penalty.get_value(borrow=True) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
182 except: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
183 print trainer.effective_l1_penalty |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
184 |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
185 print 'U min max', rbm.U.get_value(borrow=True).min(), rbm.U.get_value(borrow=True).max(), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
186 print 'W min max', rbm.W.get_value(borrow=True).min(), rbm.W.get_value(borrow=True).max(), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
187 print 'a min max', rbm.a.get_value(borrow=True).min(), rbm.a.get_value(borrow=True).max(), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
188 print 'b min max', rbm.b.get_value(borrow=True).min(), rbm.b.get_value(borrow=True).max(), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
189 print 'c min max', rbm.c.get_value(borrow=True).min(), rbm.c.get_value(borrow=True).max() |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
190 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
191 if persistent_chains: |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
192 print 'parts min', smplr.positions.get_value(borrow=True).min(), |
1524 | 193 print 'max', smplr.positions.get_value(borrow=True).max(), |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
194 print 'HMC step', smplr.stepsize.get_value(borrow=True), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
195 print 'arate', smplr.avg_acceptance_rate.get_value(borrow=True) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
196 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
197 l2_of_Ugrad = learn_fn(jj) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
198 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
199 if persistent_chains and print_jj: |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
200 print 'l2(U_grad)', float(l2_of_Ugrad[0]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
201 print 'l2(U_inc)', float(l2_of_Ugrad[1]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
202 print 'l2(W_inc)', float(l2_of_Ugrad[2]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
203 #print 'FE+', float(l2_of_Ugrad[2]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
204 #print 'FE+[0]', float(l2_of_Ugrad[3]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
205 #print 'FE+[1]', float(l2_of_Ugrad[4]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
206 #print 'FE+[2]', float(l2_of_Ugrad[5]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
207 #print 'FE+[3]', float(l2_of_Ugrad[6]) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
208 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
209 if not as_unittest: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
210 if jj % 2000 == 0: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
211 print '' |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
212 print 'Saving rbm...' |
1524 | 213 cPickle.dump(rbm, open('mcRBM.rbm.%06i.pkl' % jj, 'w'), -1) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
214 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
215 print 'Saving sampler...' |
1524 | 216 cPickle.dump(smplr, open( |
217 'mcRBM.smplr.%06i.pkl' % jj, 'w'), -1) | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
218 |
1273
7bb5dd98e671
mcRBM - added hack to main script to pickle final model and sampler
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1272
diff
changeset
|
219 if not as_unittest: |
7bb5dd98e671
mcRBM - added hack to main script to pickle final model and sampler
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1272
diff
changeset
|
220 return rbm, smplr |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
221 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
222 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
223 def run_classif_experiment(checkpoint): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
224 |
1524 | 225 R, C = 8, 8 |
226 n_vis = 74 | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
227 # PRETRAIN |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
228 # |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
229 # extract 1 million 8x8 patches from TinyImages |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
230 # pre-process them the right way |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
231 # find 74 dims of PCA |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
232 # filter patches through PCA |
1524 | 233 whitened_patches, pca_dct = pylearn.dataset_ops.tinyimages.main( |
234 n_imgs=100000, max_components=n_vis, seed=234) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
235 # |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
236 # Set up mcRBM Trainer |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
237 # Initialize P using topological 3x3 overlapping patches thing |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
238 # start learning P matrix after 2 passes through dataset |
1509 | 239 # |
1524 | 240 rbm_filename = 'mcRBM.rbm.%06i.pkl' % 46000 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
241 try: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
242 open(rbm_filename).close() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
243 load_mcrbm = True |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
244 except: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
245 load_mcrbm = False |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
246 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
247 if load_mcrbm: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
248 print 'loading mcRBM from file', rbm_filename |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
249 rbm = cPickle.load(open(rbm_filename)) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
250 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
251 else: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
252 print "Training mcRBM" |
1524 | 253 batchsize = 128 |
254 epoch_size = len(whitened_patches) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
255 tile = pylearn.dataset_ops.tinyimages.save_filters |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
256 train_batch = theano.tensor.matrix() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
257 trainer = mcRBMTrainer.alloc_for_P( |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
258 rbm=mcRBM_withP.alloc_topo_P(n_I=n_vis, n_J=81), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
259 visible_batch=train_batch, |
1509 | 260 batchsize=batchsize, |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
261 initial_lr_per_example=0.05, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
262 l1_penalty=1e-3, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
263 l1_penalty_start=sys.maxint, |
1524 | 264 p_training_start=2 * epoch_size // batchsize, |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
265 persistent_chains=False) |
1524 | 266 rbm = trainer.rbm |
267 learn_fn = theano.function([train_batch], outputs=[], | |
268 updates=trainer.cd_updates()) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
269 smplr = trainer._last_cd1_sampler |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
270 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
271 ii = 0 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
272 for i_epoch in range(6): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
273 for i_batch in xrange(epoch_size // batchsize): |
1524 | 274 batch_vals = whitened_patches[i_batch * |
275 batchsize:(i_batch + 1) * batchsize] | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
276 learn_fn(batch_vals) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
277 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
278 if (ii % 1000) == 0: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
279 #tile(imgs_fn(ii), "imgs_%06i.png"%ii) |
1524 | 280 tile(rbm.U.value.T, "U_%06i.png" % ii) |
281 tile(rbm.W.value.T, "W_%06i.png" % ii) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
282 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
283 print 'saving samples', ii, 'epoch', i_epoch, i_batch |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
284 |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
285 print 'l2(U)', l2(rbm.U.get_value(borrow=True)), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
286 print 'l2(W)', l2(rbm.W.get_value(borrow=True)), |
1509 | 287 print 'l1_penalty', |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
288 try: |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
289 print trainer.effective_l1_penalty.get_value(borrow=True) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
290 except: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
291 print trainer.effective_l1_penalty |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
292 |
1524 | 293 print 'U min max', rbm.U.get_value( |
294 borrow=True).min(), rbm.U.get_value(borrow=True).max(), | |
295 print 'W min max', rbm.W.get_value( | |
296 borrow=True).min(), rbm.W.get_value(borrow=True).max(), | |
297 print 'a min max', rbm.a.get_value( | |
298 borrow=True).min(), rbm.a.get_value(borrow=True).max(), | |
299 print 'b min max', rbm.b.get_value( | |
300 borrow=True).min(), rbm.b.get_value(borrow=True).max(), | |
301 print 'c min max', rbm.c.get_value( | |
302 borrow=True).min(), rbm.c.get_value(borrow=True).max() | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
303 |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
304 print 'HMC step', smplr.stepsize.get_value(borrow=True), |
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
305 print 'arate', smplr.avg_acceptance_rate.get_value(borrow=True) |
1524 | 306 print 'P min max', rbm.P.get_value( |
307 borrow=True).min(), rbm.P.get_value(borrow=True).max(), | |
1512
7f166d01bf8e
Remove deprecation warning.
Frederic Bastien <nouiz@nouiz.org>
parents:
1509
diff
changeset
|
308 print 'P_lr', trainer.p_lr.get_value(borrow=True) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
309 print '' |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
310 print 'Saving rbm...' |
1524 | 311 cPickle.dump(rbm, open('mcRBM.rbm.%06i.pkl' % ii, 'w'), -1) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
312 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
313 ii += 1 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
314 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
315 # extract convolutional features from the CIFAR10 data |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
316 feat_filename = 'mcrbm_features.npy' |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
317 feat_filename = 'cifar10.features.46000.npy' |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
318 try: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
319 open(feat_filename).close() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
320 load_features = True |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
321 except: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
322 load_features = False |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
323 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
324 if load_features: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
325 print 'Loading features from', feat_filename |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
326 all_features = numpy.load(feat_filename, mmap_mode='r') |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
327 else: |
1524 | 328 batchsize = 100 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
329 feat_idx = tensor.lscalar() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
330 feat_idx_range = feat_idx * batchsize + tensor.arange(batchsize) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
331 train_batch_x, train_batch_y = pylearn.dataset_ops.cifar10.cifar10( |
1509 | 332 feat_idx_range, |
333 split='all', | |
334 dtype='uint8', | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
335 rasterized=False, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
336 color='rgb') |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
337 |
1524 | 338 WINDOW_SIZE = 8 |
339 WINDOW_STRIDE = 4 | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
340 |
1524 | 341 # put these into shared vars because support for big matrix |
342 # constants is bad, (comparing them is slow) | |
1507 | 343 pca_eigvecs = theano.shared(pca_dct['eig_vecs'].astype('float32')) |
344 pca_eigvals = theano.shared(pca_dct['eig_vals'].astype('float32')) | |
1524 | 345 pca_mean = theano.shared(pca_dct['mean'].astype('float32')) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
346 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
347 def theano_pca_whiten(X): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
348 #copying preprepcessing.pca.pca_whiten |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
349 return tensor.true_div( |
1524 | 350 tensor.dot(X - pca_mean, pca_eigvecs), |
351 tensor.sqrt(pca_eigvals) + 1e-8) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
352 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
353 h_list = [] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
354 g_list = [] |
1524 | 355 for r_offset in range(0, 32 - WINDOW_SIZE + 1, WINDOW_STRIDE): |
356 for c_offset in range(0, 32 - WINDOW_SIZE + 1, WINDOW_STRIDE): | |
357 window = train_batch_x[:, r_offset:r_offset + WINDOW_SIZE, | |
358 c_offset:c_offset + WINDOW_SIZE, :] | |
359 assert window.dtype == 'uint8' | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
360 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
361 #rasterize the patches |
1524 | 362 raster_window = tensor.flatten(tensor.cast( |
363 window, 'float32'), 2) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
364 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
365 #subtract off the mean of each image |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
366 raster_window = raster_window - raster_window.mean(axis=1).reshape((batchsize,1)) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
367 |
1524 | 368 h, g = rbm.expected_h_g_given_v( |
369 theano_pca_whiten(raster_window)) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
370 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
371 h_list.append(h) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
372 g_list.append(g) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
373 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
374 hg = tensor.concatenate(h_list + g_list, axis=1) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
375 |
1507 | 376 feat_fn = theano.function([feat_idx], hg) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
377 features = numpy.empty((60000, 11025), dtype='float32') |
1524 | 378 for i in xrange(60000 // batchsize): |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
379 if i % 100 == 0: |
1524 | 380 print("feature batch %i" % i) |
381 features[i * batchsize:(i + 1) * batchsize] = feat_fn(i) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
382 |
1524 | 383 print("saving features to %s" % feat_filename) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
384 numpy.save(feat_filename, features) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
385 all_features = features |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
386 del features |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
387 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
388 # CLASSIFY FEATURES |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
389 if 0: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
390 # nothing to load |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
391 pass |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
392 else: |
1524 | 393 batchsize = 100 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
394 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
395 if feat_filename.startswith('cifar'): |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
396 learnrate = 0.002 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
397 l1_regularization = 0.004 |
1524 | 398 anneal_epoch = 100 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
399 n_epochs = 500 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
400 else: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
401 learnrate = 0.005 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
402 l1_regularization = 0.004 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
403 n_epochs = 100 |
1524 | 404 anneal_epoch = 20 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
405 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
406 x_i = tensor.matrix() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
407 y_i = tensor.ivector() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
408 lr = tensor.scalar() |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
409 #l1_regularization = float(sys.argv[1]) #1.e-3 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
410 #l2_regularization = float(sys.argv[2]) #1.e-3*0 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
411 |
1509 | 412 feature_logreg = LogisticRegression.new(x_i, |
1524 | 413 n_in=11025, n_out=10, |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
414 dtype=x_i.dtype) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
415 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
416 # marc'aurelio does this... |
1524 | 417 feature_logreg.w.value = numpy.random.RandomState(44).randn(11025, |
418 10) * .02 | |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
419 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
420 traincost = feature_logreg.nll(y_i).sum() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
421 traincost = traincost + abs(feature_logreg.w).sum() * l1_regularization |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
422 #traincost = traincost + (feature_logreg.w**2).sum() * l2_regularization |
1509 | 423 train_logreg_fn = theano.function([x_i, y_i, lr], |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
424 [feature_logreg.nll(y_i).mean(), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
425 feature_logreg.errors(y_i).mean()], |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
426 updates=pylearn.gd.sgd.sgd_updates( |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
427 params=feature_logreg.params, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
428 grads=tensor.grad(traincost, feature_logreg.params), |
1524 | 429 stepsizes=[lr, lr / 10.])) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
430 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
431 all_labels = pylearn.dataset_ops.cifar10.all_data_labels('uint8')[1] |
1524 | 432 pylearn.dataset_ops.cifar10.all_data_labels.forget() |
433 # clear memo cache | |
434 assert len(all_labels) == 60000 | |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
435 if 0: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
436 print "Using validation set" |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
437 train_labels = all_labels[:40000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
438 valid_labels = all_labels[40000:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
439 test_labels = all_labels[50000:60000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
440 train_features = all_features[:40000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
441 valid_features = all_features[40000:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
442 test_features = all_features[50000:60000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
443 else: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
444 print "NOT USING validation set" |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
445 train_labels = all_labels[:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
446 valid_labels = None |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
447 test_labels = all_labels[50000:60000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
448 train_features = all_features[:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
449 valid_features = None |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
450 test_features = all_features[50000:60000] |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
451 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
452 if 1: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
453 print "Computing mean and std.dev" |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
454 train_mean = train_features.mean(axis=0) |
1524 | 455 train_std = train_features.std(axis=0) + 1e-4 |
456 preproc = lambda x: (x - train_mean) / (0.1 + train_std) | |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
457 else: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
458 print "Not centering data" |
1524 | 459 preproc = lambda x: x |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
460 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
461 for epoch in xrange(n_epochs): |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
462 print 'epoch', epoch |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
463 # validate |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
464 # Marc'Aurelio, you crazy!! |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
465 # the division by batchsize is done in the cost function |
1524 | 466 e_lr = learnrate / (batchsize * max(1.0, numpy.floor( |
467 max(1., epoch / float(anneal_epoch)) - 2))) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
468 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
469 if valid_features is not None: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
470 l01s = [] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
471 nlls = [] |
1524 | 472 for i in xrange(10000 / batchsize): |
473 x_i = valid_features[i * batchsize:(i + 1) * batchsize] | |
474 y_i = valid_labels[i * batchsize:(i + 1) * batchsize] | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
475 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
476 #lr=0.0 -> no learning, safe for validation set |
1509 | 477 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0) |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
478 nlls.append(nll) |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
479 l01s.append(l01) |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
480 print 'validate log_reg', numpy.mean(nlls), numpy.mean(l01s) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
481 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
482 # test |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
483 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
484 l01s = [] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
485 nlls = [] |
1524 | 486 for i in xrange(len(test_features) // batchsize): |
487 x_i = test_features[i * batchsize:(i + 1) * batchsize] | |
488 y_i = test_labels[i * batchsize:(i + 1) * batchsize] | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
489 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
490 #lr=0.0 -> no learning, safe for validation set |
1509 | 491 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
492 nlls.append(nll) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
493 l01s.append(l01) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
494 print 'test log_reg', numpy.mean(nlls), numpy.mean(l01s) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
495 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
496 #train |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
497 l01s = [] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
498 nlls = [] |
1524 | 499 for i in xrange(len(train_features) // batchsize): |
500 x_i = train_features[i * batchsize:(i + 1) * batchsize] | |
501 y_i = train_labels[i * batchsize:(i + 1) * batchsize] | |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
502 nll, l01 = train_logreg_fn(preproc(x_i), y_i, e_lr) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
503 nlls.append(nll) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
504 l01s.append(l01) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
505 print 'train log_reg', numpy.mean(nlls), numpy.mean(l01s) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
506 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
507 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
508 import pickle as cPickle |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
509 #import cPickle |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
510 if __name__ == '__main__': |
1509 | 511 if 0: |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
512 #learning 16 x 16 pinwheel filters from official cifar patches (MAR) |
1524 | 513 rbm, smplr = test_reproduce_ranzato_hinton_2010( |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
514 as_unittest=False, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
515 n_train_iters=5000, |
1524 | 516 rbm_alloc=lambda n_I: mcRBM_withP.alloc_topo_P(n_I, n_J=81), |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
517 trainer_alloc=mcRBMTrainer.alloc_for_P, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
518 dataset='MAR' |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
519 ) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
520 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
521 if 0: |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
522 # pretraining settings |
1524 | 523 rbm, smplr = test_reproduce_ranzato_hinton_2010( |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
524 as_unittest=False, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
525 n_train_iters=60000, |
1524 | 526 rbm_alloc=lambda n_I: mcRBM_withP.alloc_topo_P(n_I, n_J=81), |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
527 trainer_alloc=mcRBMTrainer.alloc_for_P, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
528 lr_per_example=0.05, |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
529 dataset='tinyimages_patches', |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
530 l1_penalty=1e-3, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
531 l1_penalty_start=30000, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
532 #l1_penalty_start=350, #DEBUG |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
533 persistent_chains=False |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
534 ) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
535 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
536 if 1: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
537 def checkpoint(): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
538 return checkpoint |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
539 run_classif_experiment(checkpoint=checkpoint) |