Mercurial > pylearn
annotate pylearn/algorithms/tests/test_mcRBM.py @ 1508:b28e8730c948
fix test.
author | Frederic Bastien <nouiz@nouiz.org> |
---|---|
date | Mon, 12 Sep 2011 11:45:56 -0400 |
parents | 2a6a6f16416c |
children | b709f6b53b17 |
rev | line source |
---|---|
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
1 import sys |
1507 | 2 |
3 import numpy | |
4 import theano | |
5 from theano import tensor | |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
6 |
1507 | 7 from pylearn.algorithms.mcRBM import mcRBM, mcRBMTrainer, mcRBM_withP, l2 |
8 #import pylearn.datasets.cifar10 | |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
9 import pylearn.dataset_ops.cifar10 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
10 from pylearn.shared.layers.logreg import LogisticRegression |
1507 | 11 from pylearn.io import image_tiling |
12 import pylearn.dataset_ops.image_patches | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
13 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
14 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
15 def _default_rbm_alloc(n_I, n_K=256, n_J=100): |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
16 return mcRBM.alloc(n_I, n_K, n_J) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
17 |
1508 | 18 def _default_trainer_alloc(rbm, train_batch, batchsize, initial_lr_per_example, |
19 l1_penalty, l1_penalty_start, persistent_chains): | |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
20 return mcRBMTrainer.alloc(rbm, train_batch, batchsize, l1_penalty=l1_penalty, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
21 l1_penalty_start=l1_penalty_start,persistent_chains=persistent_chains) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
22 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
23 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
24 def test_reproduce_ranzato_hinton_2010(dataset='MAR', as_unittest=True, n_train_iters=5000, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
25 rbm_alloc=_default_rbm_alloc, trainer_alloc=_default_trainer_alloc, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
26 lr_per_example=.075, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
27 l1_penalty=1e-3, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
28 l1_penalty_start=1000, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
29 persistent_chains=True, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
30 ): |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
31 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
32 batchsize = 128 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
33 |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
34 if dataset == 'MAR': |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
35 n_vis=105 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
36 n_patches=10240 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
37 epoch_size=n_patches |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
38 elif dataset=='cifar10patches8x8': |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
39 R,C= 8,8 # the size of image patches |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
40 n_vis=96 # pca components |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
41 epoch_size=batchsize*500 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
42 n_patches=epoch_size*20 |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
43 elif dataset=='tinyimages_patches': |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
44 R,C=8,8 |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
45 n_vis=81 |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
46 epoch_size=batchsize*500 |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
47 n_patches=epoch_size*20 |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
48 else: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
49 R,C= 16,16 # the size of image patches |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
50 n_vis=R*C |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
51 n_patches=100000 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
52 epoch_size=n_patches |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
53 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
54 def l2(X): |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
55 return numpy.sqrt((X**2).sum()) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
56 |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
57 if dataset == 'MAR': |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
58 tile = pylearn.dataset_ops.image_patches.save_filters_of_ranzato_hinton_2010 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
59 elif dataset == 'cifar10patches8x8': |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
60 def tile(X, fname): |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
61 _img = pylearn.datasets.cifar10.tile_rasterized_examples( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
62 pylearn.preprocessing.pca.pca_whiten_inverse( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
63 pylearn.dataset_ops.cifar10.random_cifar_patches_pca( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
64 n_vis, None, 'float32', n_patches, R, C,), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
65 X), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
66 img_shape=(R,C)) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
67 image_tiling.save_tiled_raster_images(_img, fname) |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
68 elif dataset == 'tinyimages_patches': |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
69 tile = pylearn.dataset_ops.tinyimages.save_filters |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
70 else: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
71 def tile(X, fname): |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
72 _img = image_tiling.tile_raster_images(X, |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
73 img_shape=(R,C), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
74 min_dynamic_range=1e-2) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
75 image_tiling.save_tiled_raster_images(_img, fname) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
76 |
1507 | 77 batch_idx = tensor.iscalar() |
78 batch_range =batch_idx * batchsize + numpy.arange(batchsize) | |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
79 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
80 if dataset == 'MAR': |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
81 train_batch = pylearn.dataset_ops.image_patches.ranzato_hinton_2010_op(batch_range) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
82 elif dataset == 'cifar10patches8x8': |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
83 train_batch = pylearn.dataset_ops.cifar10.cifar10_patches( |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
84 batch_range, 'train', n_patches=n_patches, patch_size=(R,C), |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
85 pca_components=n_vis) |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
86 elif dataset == 'tinyimages_patches': |
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
87 train_batch = pylearn.dataset_ops.tinyimages.tinydataset_op(batch_range) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
88 else: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
89 train_batch = pylearn.dataset_ops.image_patches.image_patches( |
1507 | 90 s_idx = (batch_idx * batchsize + numpy.arange(batchsize)), |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
91 dims = (n_patches,R,C), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
92 center=True, |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
93 unitvar=True, |
1507 | 94 dtype=theano.config.floatX, |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
95 rasterized=True) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
96 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
97 if not as_unittest: |
1507 | 98 imgs_fn = theano.function([batch_idx], outputs=train_batch) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
99 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
100 trainer = trainer_alloc( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
101 rbm_alloc(n_I=n_vis), |
1267
075c193afd1b
refactoring mcRBM
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1000
diff
changeset
|
102 train_batch, |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
103 batchsize, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
104 initial_lr_per_example=lr_per_example, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
105 l1_penalty=l1_penalty, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
106 l1_penalty_start=l1_penalty_start, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
107 persistent_chains=persistent_chains) |
1267
075c193afd1b
refactoring mcRBM
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1000
diff
changeset
|
108 rbm=trainer.rbm |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
109 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
110 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
111 grads = trainer.contrastive_grads() |
1507 | 112 learn_fn = theano.function([batch_idx], |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
113 outputs=[grads[0].norm(2), grads[0].norm(2), grads[1].norm(2)], |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
114 updates=trainer.cd_updates()) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
115 else: |
1507 | 116 learn_fn = theano.function([batch_idx], outputs=[], updates=trainer.cd_updates()) |
1267
075c193afd1b
refactoring mcRBM
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1000
diff
changeset
|
117 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
118 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
119 smplr = trainer.sampler |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
120 else: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
121 smplr = trainer._last_cd1_sampler |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
122 |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
123 if dataset == 'cifar10patches8x8': |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
124 cPickle.dump( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
125 pylearn.dataset_ops.cifar10.random_cifar_patches_pca( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
126 n_vis, None, 'float32', n_patches, R, C,), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
127 open('test_mcRBM.pca.pkl','w')) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
128 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
129 print "Learning..." |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
130 last_epoch = -1 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
131 for jj in xrange(n_train_iters): |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
132 epoch = jj*batchsize / epoch_size |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
133 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
134 print_jj = epoch != last_epoch |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
135 last_epoch = epoch |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
136 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
137 if as_unittest and epoch == 5: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
138 U = rbm.U.value |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
139 W = rbm.W.value |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
140 def allclose(a,b): |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
141 return numpy.allclose(a,b,rtol=1.01,atol=1e-3) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
142 print "" |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
143 print "--------------" |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
144 print "assert allclose(l2(U), %f)"%l2(U) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
145 print "assert allclose(l2(W), %f)"%l2(W) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
146 print "assert allclose(U.min(), %f)"%U.min() |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
147 print "assert allclose(U.max(), %f)"%U.max() |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
148 print "assert allclose(W.min(),%f)"%W.min() |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
149 print "assert allclose(W.max(), %f)"%W.max() |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
150 print "--------------" |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
151 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
152 assert allclose(l2(U), 21.351664) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
153 assert allclose(l2(W), 6.275828) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
154 assert allclose(U.min(), -1.176703) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
155 assert allclose(U.max(), 0.859802) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
156 assert allclose(W.min(),-0.223128) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
157 assert allclose(W.max(), 0.227558 ) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
158 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
159 break |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
160 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
161 if print_jj: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
162 if not as_unittest: |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
163 tile(imgs_fn(jj), "imgs_%06i.png"%jj) |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
164 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
165 tile(smplr.positions.value, "sample_%06i.png"%jj) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
166 tile(rbm.U.value.T, "U_%06i.png"%jj) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
167 tile(rbm.W.value.T, "W_%06i.png"%jj) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
168 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
169 print 'saving samples', jj, 'epoch', jj/(epoch_size/batchsize) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
170 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
171 print 'l2(U)', l2(rbm.U.value), |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
172 print 'l2(W)', l2(rbm.W.value), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
173 print 'l1_penalty', |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
174 try: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
175 print trainer.effective_l1_penalty.value |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
176 except: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
177 print trainer.effective_l1_penalty |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
178 |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
179 print 'U min max', rbm.U.value.min(), rbm.U.value.max(), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
180 print 'W min max', rbm.W.value.min(), rbm.W.value.max(), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
181 print 'a min max', rbm.a.value.min(), rbm.a.value.max(), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
182 print 'b min max', rbm.b.value.min(), rbm.b.value.max(), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
183 print 'c min max', rbm.c.value.min(), rbm.c.value.max() |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
184 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
185 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
186 print 'parts min', smplr.positions.value.min(), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
187 print 'max',smplr.positions.value.max(), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
188 print 'HMC step', smplr.stepsize.value, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
189 print 'arate', smplr.avg_acceptance_rate.value |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
190 |
1267
075c193afd1b
refactoring mcRBM
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1000
diff
changeset
|
191 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
192 l2_of_Ugrad = learn_fn(jj) |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
193 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
194 if persistent_chains and print_jj: |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
195 print 'l2(U_grad)', float(l2_of_Ugrad[0]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
196 print 'l2(U_inc)', float(l2_of_Ugrad[1]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
197 print 'l2(W_inc)', float(l2_of_Ugrad[2]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
198 #print 'FE+', float(l2_of_Ugrad[2]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
199 #print 'FE+[0]', float(l2_of_Ugrad[3]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
200 #print 'FE+[1]', float(l2_of_Ugrad[4]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
201 #print 'FE+[2]', float(l2_of_Ugrad[5]), |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
202 #print 'FE+[3]', float(l2_of_Ugrad[6]) |
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
203 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
204 if not as_unittest: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
205 if jj % 2000 == 0: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
206 print '' |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
207 print 'Saving rbm...' |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
208 cPickle.dump(rbm, open('mcRBM.rbm.%06i.pkl'%jj, 'w'), -1) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
209 if persistent_chains: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
210 print 'Saving sampler...' |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
211 cPickle.dump(smplr, open('mcRBM.smplr.%06i.pkl'%jj, 'w'), -1) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
212 |
1000
d4a14c6c36e0
mcRBM - post code-review #1 with Guillaume
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
diff
changeset
|
213 |
1273
7bb5dd98e671
mcRBM - added hack to main script to pickle final model and sampler
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1272
diff
changeset
|
214 if not as_unittest: |
7bb5dd98e671
mcRBM - added hack to main script to pickle final model and sampler
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1272
diff
changeset
|
215 return rbm, smplr |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
216 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
217 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
218 def run_classif_experiment(checkpoint): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
219 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
220 R,C=8,8 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
221 n_vis=74 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
222 # PRETRAIN |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
223 # |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
224 # extract 1 million 8x8 patches from TinyImages |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
225 # pre-process them the right way |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
226 # find 74 dims of PCA |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
227 # filter patches through PCA |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
228 whitened_patches, pca_dct = pylearn.dataset_ops.tinyimages.main(n_imgs=100000, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
229 max_components=n_vis, seed=234) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
230 # |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
231 # Set up mcRBM Trainer |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
232 # Initialize P using topological 3x3 overlapping patches thing |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
233 # start learning P matrix after 2 passes through dataset |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
234 # |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
235 rbm_filename = 'mcRBM.rbm.%06i.pkl'%46000 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
236 try: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
237 open(rbm_filename).close() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
238 load_mcrbm = True |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
239 except: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
240 load_mcrbm = False |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
241 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
242 if load_mcrbm: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
243 print 'loading mcRBM from file', rbm_filename |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
244 rbm = cPickle.load(open(rbm_filename)) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
245 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
246 else: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
247 print "Training mcRBM" |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
248 batchsize=128 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
249 epoch_size=len(whitened_patches) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
250 tile = pylearn.dataset_ops.tinyimages.save_filters |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
251 train_batch = theano.tensor.matrix() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
252 trainer = mcRBMTrainer.alloc_for_P( |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
253 rbm=mcRBM_withP.alloc_topo_P(n_I=n_vis, n_J=81), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
254 visible_batch=train_batch, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
255 batchsize=batchsize, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
256 initial_lr_per_example=0.05, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
257 l1_penalty=1e-3, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
258 l1_penalty_start=sys.maxint, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
259 p_training_start=2*epoch_size//batchsize, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
260 persistent_chains=False) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
261 rbm=trainer.rbm |
1507 | 262 learn_fn = theano.function([train_batch], outputs=[], updates=trainer.cd_updates()) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
263 smplr = trainer._last_cd1_sampler |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
264 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
265 ii = 0 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
266 for i_epoch in range(6): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
267 for i_batch in xrange(epoch_size // batchsize): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
268 batch_vals = whitened_patches[i_batch*batchsize:(i_batch+1)*batchsize] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
269 learn_fn(batch_vals) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
270 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
271 if (ii % 1000) == 0: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
272 #tile(imgs_fn(ii), "imgs_%06i.png"%ii) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
273 tile(rbm.U.value.T, "U_%06i.png"%ii) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
274 tile(rbm.W.value.T, "W_%06i.png"%ii) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
275 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
276 print 'saving samples', ii, 'epoch', i_epoch, i_batch |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
277 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
278 print 'l2(U)', l2(rbm.U.value), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
279 print 'l2(W)', l2(rbm.W.value), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
280 print 'l1_penalty', |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
281 try: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
282 print trainer.effective_l1_penalty.value |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
283 except: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
284 print trainer.effective_l1_penalty |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
285 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
286 print 'U min max', rbm.U.value.min(), rbm.U.value.max(), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
287 print 'W min max', rbm.W.value.min(), rbm.W.value.max(), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
288 print 'a min max', rbm.a.value.min(), rbm.a.value.max(), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
289 print 'b min max', rbm.b.value.min(), rbm.b.value.max(), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
290 print 'c min max', rbm.c.value.min(), rbm.c.value.max() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
291 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
292 print 'HMC step', smplr.stepsize.value, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
293 print 'arate', smplr.avg_acceptance_rate.value |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
294 print 'P min max', rbm.P.value.min(), rbm.P.value.max(), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
295 print 'P_lr', trainer.p_lr.value |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
296 print '' |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
297 print 'Saving rbm...' |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
298 cPickle.dump(rbm, open('mcRBM.rbm.%06i.pkl'%ii, 'w'), -1) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
299 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
300 ii += 1 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
301 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
302 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
303 # extract convolutional features from the CIFAR10 data |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
304 feat_filename = 'mcrbm_features.npy' |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
305 feat_filename = 'cifar10.features.46000.npy' |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
306 try: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
307 open(feat_filename).close() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
308 load_features = True |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
309 except: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
310 load_features = False |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
311 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
312 if load_features: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
313 print 'Loading features from', feat_filename |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
314 all_features = numpy.load(feat_filename, mmap_mode='r') |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
315 else: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
316 batchsize=100 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
317 feat_idx = tensor.lscalar() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
318 feat_idx_range = feat_idx * batchsize + tensor.arange(batchsize) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
319 train_batch_x, train_batch_y = pylearn.dataset_ops.cifar10.cifar10( |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
320 feat_idx_range, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
321 split='all', |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
322 dtype='uint8', |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
323 rasterized=False, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
324 color='rgb') |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
325 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
326 WINDOW_SIZE=8 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
327 WINDOW_STRIDE=4 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
328 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
329 # put these into shared vars because support for big matrix constants is bad, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
330 # (comparing them is slow) |
1507 | 331 pca_eigvecs = theano.shared(pca_dct['eig_vecs'].astype('float32')) |
332 pca_eigvals = theano.shared(pca_dct['eig_vals'].astype('float32')) | |
333 pca_mean = theano.shared(pca_dct['mean'].astype('float32')) | |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
334 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
335 def theano_pca_whiten(X): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
336 #copying preprepcessing.pca.pca_whiten |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
337 return tensor.true_div( |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
338 tensor.dot(X-pca_mean, pca_eigvecs), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
339 tensor.sqrt(pca_eigvals)+1e-8) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
340 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
341 h_list = [] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
342 g_list = [] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
343 for r_offset in range(0, 32-WINDOW_SIZE+1, WINDOW_STRIDE): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
344 for c_offset in range(0, 32-WINDOW_SIZE+1, WINDOW_STRIDE): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
345 window = train_batch_x[:, r_offset:r_offset+WINDOW_SIZE, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
346 c_offset:c_offset+WINDOW_SIZE, :] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
347 assert window.dtype=='uint8' |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
348 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
349 #rasterize the patches |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
350 raster_window = tensor.flatten(tensor.cast(window, 'float32'),2) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
351 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
352 #subtract off the mean of each image |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
353 raster_window = raster_window - raster_window.mean(axis=1).reshape((batchsize,1)) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
354 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
355 h,g = rbm.expected_h_g_given_v(theano_pca_whiten(raster_window)) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
356 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
357 h_list.append(h) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
358 g_list.append(g) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
359 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
360 hg = tensor.concatenate(h_list + g_list, axis=1) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
361 |
1507 | 362 feat_fn = theano.function([feat_idx], hg) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
363 features = numpy.empty((60000, 11025), dtype='float32') |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
364 for i in xrange(60000//batchsize): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
365 if i % 100 == 0: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
366 print("feature batch %i"%i) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
367 features[i*batchsize:(i+1)*batchsize] = feat_fn(i) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
368 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
369 print("saving features to %s"%feat_filename) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
370 numpy.save(feat_filename, features) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
371 all_features = features |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
372 del features |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
373 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
374 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
375 # CLASSIFY FEATURES |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
376 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
377 if 0: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
378 # nothing to load |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
379 pass |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
380 else: |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
381 batchsize=100 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
382 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
383 if feat_filename.startswith('cifar'): |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
384 learnrate = 0.002 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
385 l1_regularization = 0.004 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
386 anneal_epoch=100 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
387 n_epochs = 500 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
388 else: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
389 learnrate = 0.005 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
390 l1_regularization = 0.004 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
391 n_epochs = 100 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
392 anneal_epoch=20 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
393 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
394 x_i = tensor.matrix() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
395 y_i = tensor.ivector() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
396 lr = tensor.scalar() |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
397 #l1_regularization = float(sys.argv[1]) #1.e-3 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
398 #l2_regularization = float(sys.argv[2]) #1.e-3*0 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
399 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
400 feature_logreg = LogisticRegression.new(x_i, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
401 n_in = 11025, n_out=10, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
402 dtype=x_i.dtype) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
403 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
404 # marc'aurelio does this... |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
405 feature_logreg.w.value = numpy.random.RandomState(44).randn(11025,10)*.02 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
406 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
407 traincost = feature_logreg.nll(y_i).sum() |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
408 traincost = traincost + abs(feature_logreg.w).sum() * l1_regularization |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
409 #traincost = traincost + (feature_logreg.w**2).sum() * l2_regularization |
1507 | 410 train_logreg_fn = theano.function([x_i, y_i, lr], |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
411 [feature_logreg.nll(y_i).mean(), |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
412 feature_logreg.errors(y_i).mean()], |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
413 updates=pylearn.gd.sgd.sgd_updates( |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
414 params=feature_logreg.params, |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
415 grads=tensor.grad(traincost, feature_logreg.params), |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
416 stepsizes=[lr,lr/10.])) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
417 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
418 all_labels = pylearn.dataset_ops.cifar10.all_data_labels('uint8')[1] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
419 pylearn.dataset_ops.cifar10.all_data_labels.forget() # clear memo cache |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
420 assert len(all_labels)==60000 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
421 if 0: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
422 print "Using validation set" |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
423 train_labels = all_labels[:40000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
424 valid_labels = all_labels[40000:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
425 test_labels = all_labels[50000:60000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
426 train_features = all_features[:40000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
427 valid_features = all_features[40000:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
428 test_features = all_features[50000:60000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
429 else: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
430 print "NOT USING validation set" |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
431 train_labels = all_labels[:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
432 valid_labels = None |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
433 test_labels = all_labels[50000:60000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
434 train_features = all_features[:50000] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
435 valid_features = None |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
436 test_features = all_features[50000:60000] |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
437 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
438 if 1: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
439 print "Computing mean and std.dev" |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
440 train_mean = train_features.mean(axis=0) |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
441 train_std = train_features.std(axis=0)+1e-4 |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
442 preproc = lambda x: (x-train_mean)/(0.1+train_std) |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
443 else: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
444 print "Not centering data" |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
445 preproc = lambda x:x |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
446 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
447 for epoch in xrange(n_epochs): |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
448 print 'epoch', epoch |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
449 # validate |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
450 # Marc'Aurelio, you crazy!! |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
451 # the division by batchsize is done in the cost function |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
452 e_lr = learnrate / (batchsize*max(1.0, numpy.floor(max(1., epoch/float(anneal_epoch))-2))) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
453 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
454 if valid_features is not None: |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
455 l01s = [] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
456 nlls = [] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
457 for i in xrange(10000/batchsize): |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
458 x_i = valid_features[i*batchsize:(i+1)*batchsize] |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
459 y_i = valid_labels[i*batchsize:(i+1)*batchsize] |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
460 |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
461 #lr=0.0 -> no learning, safe for validation set |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
462 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0) |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
463 nlls.append(nll) |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
464 l01s.append(l01) |
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
465 print 'validate log_reg', numpy.mean(nlls), numpy.mean(l01s) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
466 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
467 # test |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
468 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
469 l01s = [] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
470 nlls = [] |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
471 for i in xrange(len(test_features)//batchsize): |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
472 x_i = test_features[i*batchsize:(i+1)*batchsize] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
473 y_i = test_labels[i*batchsize:(i+1)*batchsize] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
474 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
475 #lr=0.0 -> no learning, safe for validation set |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
476 nll, l01 = train_logreg_fn(preproc(x_i), y_i, 0.0) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
477 nlls.append(nll) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
478 l01s.append(l01) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
479 print 'test log_reg', numpy.mean(nlls), numpy.mean(l01s) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
480 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
481 #train |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
482 l01s = [] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
483 nlls = [] |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
484 for i in xrange(len(train_features)//batchsize): |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
485 x_i = train_features[i*batchsize:(i+1)*batchsize] |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
486 y_i = train_labels[i*batchsize:(i+1)*batchsize] |
1333
c7b2da4e2df6
modifs to test_mcRBM to reproduce mcRBM classif results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
487 nll, l01 = train_logreg_fn(preproc(x_i), y_i, e_lr) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
488 nlls.append(nll) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
489 l01s.append(l01) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
490 print 'train log_reg', numpy.mean(nlls), numpy.mean(l01s) |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
491 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
492 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
493 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
494 |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
495 import pickle as cPickle |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
496 #import cPickle |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
497 if __name__ == '__main__': |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
498 if 0: |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
499 #learning 16 x 16 pinwheel filters from official cifar patches (MAR) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
500 rbm,smplr = test_reproduce_ranzato_hinton_2010( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
501 as_unittest=False, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
502 n_train_iters=5000, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
503 rbm_alloc=lambda n_I : mcRBM_withP.alloc_topo_P(n_I, n_J=81), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
504 trainer_alloc=mcRBMTrainer.alloc_for_P, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
505 dataset='MAR' |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
506 ) |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
507 |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
508 if 0: |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
509 # pretraining settings |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
510 rbm,smplr = test_reproduce_ranzato_hinton_2010( |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
511 as_unittest=False, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
512 n_train_iters=60000, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
513 rbm_alloc=lambda n_I : mcRBM_withP.alloc_topo_P(n_I, n_J=81), |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
514 trainer_alloc=mcRBMTrainer.alloc_for_P, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
515 lr_per_example=0.05, |
1286
8905186b176c
test_mcRBM - added code to iterate over tinyimages
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1284
diff
changeset
|
516 dataset='tinyimages_patches', |
1284
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
517 l1_penalty=1e-3, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
518 l1_penalty_start=30000, |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
519 #l1_penalty_start=350, #DEBUG |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
520 persistent_chains=False |
1817485d586d
mcRBM - many changes incl. adding support for pooling matrix
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1273
diff
changeset
|
521 ) |
1323
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
522 |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
523 if 1: |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
524 def checkpoint(): |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
525 return checkpoint |
d6726417cf57
adding training script for test_mcRBM to reproduce classification results
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1286
diff
changeset
|
526 run_classif_experiment(checkpoint=checkpoint) |
1332
837768915081
added test idea to test_mcRBM
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
1323
diff
changeset
|
527 |