Mercurial > ift6266
annotate deep/crbm/crbm.py @ 599:587674030c5a
added authors and emails list
author | boulanni <nicolas_boulanger@hotmail.com> |
---|---|
date | Fri, 15 Oct 2010 14:14:06 -0400 |
parents | 8d116d4a7593 |
children |
rev | line source |
---|---|
337
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
1 import sys |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
2 import os, os.path |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
3 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
4 import numpy |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
5 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
6 import theano |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
7 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
8 USING_GPU = "gpu" in theano.config.device |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
9 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
10 import theano.tensor as T |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
11 from theano.tensor.nnet import conv, sigmoid |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
12 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
13 if not USING_GPU: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
14 from theano.tensor.shared_randomstreams import RandomStreams |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
15 else: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
16 from theano.sandbox.rng_mrg import MRG_RandomStreams |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
17 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
18 _PRINT_GRAPHS = True |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
19 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
20 def _init_conv_biases(num_filters, varname, rng=numpy.random): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
21 b_shp = (num_filters,) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
22 b = theano.shared( numpy.asarray( |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
23 rng.uniform(low=-.5, high=.5, size=b_shp), |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
24 dtype=theano.config.floatX), name=varname) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
25 return b |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
26 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
27 def _init_conv_weights(conv_params, varname, rng=numpy.random): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
28 cp = conv_params |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
29 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
30 # initialize shared variable for weights. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
31 w_shp = conv_params.as_conv2d_shape_tuple() |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
32 w_bound = numpy.sqrt(cp.num_input_planes * \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
33 cp.height_filters * cp.width_filters) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
34 W = theano.shared( numpy.asarray( |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
35 rng.uniform( |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
36 low=-1.0 / w_bound, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
37 high=1.0 / w_bound, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
38 size=w_shp), |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
39 dtype=theano.config.floatX), name=varname) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
40 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
41 return W |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
42 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
43 # Shape of W for conv2d |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
44 class ConvolutionParams: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
45 def __init__(self, num_filters, num_input_planes, height_filters, width_filters): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
46 self.num_filters = num_filters |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
47 self.num_input_planes = num_input_planes |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
48 self.height_filters = height_filters |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
49 self.width_filters = width_filters |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
50 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
51 def as_conv2d_shape_tuple(self): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
52 cp = self |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
53 return (cp.num_filters, cp.num_input_planes, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
54 cp.height_filters, cp.width_filters) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
55 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
56 class CRBM: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
57 def __init__(self, minibatch_size, image_size, conv_params, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
58 learning_rate, sparsity_lambda, sparsity_p): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
59 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
60 Parameters |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
61 ---------- |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
62 image_size |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
63 height, width |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
64 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
65 self.minibatch_size = minibatch_size |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
66 self.image_size = image_size |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
67 self.conv_params = conv_params |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
68 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
69 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
70 Dimensions: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
71 0- minibatch |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
72 1- plane/color |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
73 2- y (rows) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
74 3- x (cols) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
75 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
76 self.x = T.tensor4('x') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
77 self.h = T.tensor4('h') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
78 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
79 self.lr = theano.shared(numpy.asarray(learning_rate, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
80 dtype=theano.config.floatX)) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
81 self.sparsity_lambda = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
82 theano.shared( \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
83 numpy.asarray( \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
84 sparsity_lambda, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
85 dtype=theano.config.floatX)) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
86 self.sparsity_p = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
87 theano.shared( \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
88 numpy.asarray(sparsity_p, \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
89 dtype=theano.config.floatX)) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
90 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
91 self.numpy_rng = numpy.random.RandomState(1234) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
92 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
93 if not USING_GPU: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
94 self.theano_rng = RandomStreams(self.numpy_rng.randint(2**30)) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
95 else: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
96 self.theano_rng = MRG_RandomStreams(234, use_cuda=True) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
97 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
98 self._init_params() |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
99 self._init_functions() |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
100 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
101 def _get_visibles_shape(self): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
102 imsz = self.image_size |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
103 return (self.minibatch_size, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
104 self.conv_params.num_input_planes, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
105 imsz[0], imsz[1]) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
106 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
107 def _get_hiddens_shape(self): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
108 cp = self.conv_params |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
109 imsz = self.image_size |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
110 wf, hf = cp.height_filters, cp.width_filters |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
111 return (self.minibatch_size, cp.num_filters, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
112 imsz[0] - hf + 1, imsz[1] - wf + 1) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
113 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
114 def _init_params(self): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
115 cp = self.conv_params |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
116 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
117 self.W = _init_conv_weights(cp, 'W') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
118 self.b_h = _init_conv_biases(cp.num_filters, 'b_h') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
119 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
120 Lee09 mentions "all visible units share a single bias c" |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
121 but for upper layers it's pretty clear we need one |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
122 per plane, by symmetry |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
123 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
124 self.b_x = _init_conv_biases(cp.num_input_planes, 'b_x') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
125 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
126 self.params = [self.W, self.b_h, self.b_x] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
127 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
128 # flip filters horizontally and vertically |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
129 W_flipped = self.W[:, :, ::-1, ::-1] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
130 # also have to invert the filters/num_planes |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
131 self.W_tilde = W_flipped.dimshuffle(1,0,2,3) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
132 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
133 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
134 I_up and I_down come from the symbol used in the |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
135 Lee 2009 CRBM paper |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
136 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
137 def _I_up(self, visibles_mb): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
138 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
139 output of conv is features maps of size |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
140 image_size - filter_size + 1 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
141 The dimshuffle serves to broadcast b_h so that it |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
142 corresponds to output planes |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
143 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
144 fshp = self.conv_params.as_conv2d_shape_tuple() |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
145 return conv.conv2d(visibles_mb, self.W, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
146 filter_shape=fshp) + \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
147 self.b_h.dimshuffle('x',0,'x','x') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
148 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
149 def _I_down(self, hiddens_mb): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
150 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
151 notice border_mode='full'... we want to get |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
152 back the original size |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
153 so we get feature_map_size + filter_size - 1 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
154 The dimshuffle serves to broadcast b_x so that |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
155 it corresponds to output planes |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
156 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
157 fshp = list(self.conv_params.as_conv2d_shape_tuple()) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
158 # num_filters and num_planes swapped |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
159 fshp[0], fshp[1] = fshp[1], fshp[0] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
160 return conv.conv2d(hiddens_mb, self.W_tilde, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
161 border_mode='full',filter_shape=tuple(fshp)) + \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
162 self.b_x.dimshuffle('x',0,'x','x') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
163 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
164 def _mean_free_energy(self, visibles_mb): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
165 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
166 visibles_mb is mb_size x num_planes x h x w |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
167 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
168 we want to match the summed input planes |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
169 (second dimension, first is mb index) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
170 to respective bias terms for the visibles |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
171 The dimshuffle isn't really necessary, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
172 but I put it there for clarity. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
173 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
174 vbias_term = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
175 self.b_x.dimshuffle('x',0) * \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
176 T.sum(visibles_mb,axis=[2,3]) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
177 # now sum over term per planes, get one free energy |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
178 # contribution per element of minibatch |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
179 vbias_term = - T.sum(vbias_term, axis=1) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
180 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
181 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
182 Here it's a bit more complex, a few points: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
183 - The usual free energy, in the fully connected case, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
184 is a sum over all hiddens. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
185 We do the same thing here, but each unit has limited |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
186 connectivity and there's weight reuse. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
187 Therefore we only need to first do the convolutions |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
188 (with I_up) which gives us |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
189 what would normally be the Wx+b_h for each hidden. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
190 Once we have this, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
191 we take the log(1+exp(sum for this hidden)) elemwise |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
192 for each hidden, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
193 then we sum for all hiddens in one example of the minibatch. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
194 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
195 - Notice that we reuse the same b_h everywhere instead of |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
196 using one b per hidden, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
197 so the broadcasting for b_h done in I_up is all right. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
198 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
199 That sum is over all hiddens, so all filters |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
200 (planes of hiddens), x, and y. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
201 In the end we get one free energy contribution per |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
202 example of the minibatch. |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
203 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
204 softplused = T.log(1.0+T.exp(self._I_up(visibles_mb))) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
205 # h_sz = self._get_hiddens_shape() |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
206 # this simplifies the sum |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
207 # num_hiddens = h_sz[1] * h_sz[2] * h_sz[3] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
208 # reshaped = T.reshape(softplused, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
209 # (self.minibatch_size, num_hiddens)) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
210 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
211 # this is because the 0,1,1,1 sum pattern is not |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
212 # implemented on gpu, but the 1,0,1,1 pattern is |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
213 dimshuffled = softplused.dimshuffle(1,0,2,3) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
214 xh_and_hbias_term = - T.sum(dimshuffled, axis=[0,2,3]) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
215 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
216 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
217 both bias_term and vbias_term end up with one |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
218 contributor to free energy per minibatch |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
219 so we mean over minibatches |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
220 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
221 return T.mean(vbias_term + xh_and_hbias_term) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
222 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
223 def _init_functions(self): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
224 # propup |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
225 # b_h is broadcasted keeping in mind we want it to |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
226 # correspond to each new plane (corresponding to filters) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
227 I_up = self._I_up(self.x) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
228 # expected values for the distributions for each hidden |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
229 E_h_given_x = sigmoid(I_up) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
230 # might be needed if we ever want a version where we |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
231 # take expectations instead of samples for CD learning |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
232 self.E_h_given_x_func = theano.function([self.x], E_h_given_x) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
233 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
234 if _PRINT_GRAPHS: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
235 print "----------------------\nE_h_given_x_func" |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
236 theano.printing.debugprint(self.E_h_given_x_func) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
237 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
238 h_sample_given_x = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
239 self.theano_rng.binomial( \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
240 size = self._get_hiddens_shape(), |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
241 n = 1, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
242 p = E_h_given_x, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
243 dtype = theano.config.floatX) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
244 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
245 self.h_sample_given_x_func = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
246 theano.function([self.x], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
247 h_sample_given_x) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
248 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
249 if _PRINT_GRAPHS: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
250 print "----------------------\nh_sample_given_x_func" |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
251 theano.printing.debugprint(self.h_sample_given_x_func) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
252 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
253 # propdown |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
254 I_down = self._I_down(self.h) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
255 E_x_given_h = sigmoid(I_down) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
256 self.E_x_given_h_func = theano.function([self.h], E_x_given_h) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
257 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
258 if _PRINT_GRAPHS: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
259 print "----------------------\nE_x_given_h_func" |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
260 theano.printing.debugprint(self.E_x_given_h_func) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
261 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
262 x_sample_given_h = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
263 self.theano_rng.binomial( \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
264 size = self._get_visibles_shape(), |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
265 n = 1, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
266 p = E_x_given_h, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
267 dtype = theano.config.floatX) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
268 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
269 self.x_sample_given_h_func = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
270 theano.function([self.h], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
271 x_sample_given_h) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
272 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
273 if _PRINT_GRAPHS: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
274 print "----------------------\nx_sample_given_h_func" |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
275 theano.printing.debugprint(self.x_sample_given_h_func) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
276 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
277 ############################################## |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
278 # cd update done by grad of free energy |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
279 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
280 x_tilde = T.tensor4('x_tilde') |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
281 cd_update_cost = self._mean_free_energy(self.x) - \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
282 self._mean_free_energy(x_tilde) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
283 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
284 cd_grad = T.grad(cd_update_cost, self.params) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
285 # This is NLL minimization so we use a - |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
286 cd_updates = {self.W: self.W - self.lr * cd_grad[0], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
287 self.b_h: self.b_h - self.lr * cd_grad[1], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
288 self.b_x: self.b_x - self.lr * cd_grad[2]} |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
289 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
290 cd_returned = [cd_update_cost, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
291 cd_grad[0], cd_grad[1], cd_grad[2], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
292 self.lr * cd_grad[0], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
293 self.lr * cd_grad[1], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
294 self.lr * cd_grad[2]] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
295 self.cd_return_desc = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
296 ['cd_update_cost', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
297 'cd_grad_W', 'cd_grad_b_h', 'cd_grad_b_x', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
298 'lr_times_cd_grad_W', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
299 'lr_times_cd_grad_b_h', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
300 'lr_times_cd_grad_b_x'] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
301 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
302 self.cd_update_function = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
303 theano.function([self.x, x_tilde], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
304 cd_returned, updates=cd_updates) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
305 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
306 if _PRINT_GRAPHS: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
307 print "----------------------\ncd_update_function" |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
308 theano.printing.debugprint(self.cd_update_function) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
309 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
310 ############## |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
311 # sparsity update, based on grad for b_h only |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
312 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
313 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
314 This mean returns an array of shape |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
315 (num_hiddens_planes, feature_map_height, feature_map_width) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
316 (so it's a mean over each unit's activation) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
317 ''' |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
318 mean_expected_activation = T.mean(E_h_given_x, axis=0) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
319 # sparsity_p is broadcasted everywhere |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
320 sparsity_update_cost = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
321 T.sqr(self.sparsity_p - mean_expected_activation) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
322 sparsity_update_cost = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
323 T.sum(T.sum(T.sum( \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
324 sparsity_update_cost, axis=2), axis=1), axis=0) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
325 sparsity_grad = T.grad(sparsity_update_cost, [self.W, self.b_h]) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
326 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
327 sparsity_returned = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
328 [sparsity_update_cost, |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
329 sparsity_grad[0], sparsity_grad[1], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
330 self.sparsity_lambda * self.lr * sparsity_grad[0], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
331 self.sparsity_lambda * self.lr * sparsity_grad[1]] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
332 self.sparsity_return_desc = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
333 ['sparsity_update_cost', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
334 'sparsity_grad_W', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
335 'sparsity_grad_b_h', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
336 'lambda_lr_times_sparsity_grad_W', |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
337 'lambda_lr_times_sparsity_grad_b_h'] |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
338 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
339 # gradient _descent_ so we use a - |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
340 sparsity_update = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
341 {self.b_h: self.b_h - \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
342 self.sparsity_lambda * self.lr * sparsity_grad[1], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
343 self.W: self.W - \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
344 self.sparsity_lambda * self.lr * sparsity_grad[0]} |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
345 self.sparsity_update_function = \ |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
346 theano.function([self.x], |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
347 sparsity_returned, updates=sparsity_update) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
348 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
349 if _PRINT_GRAPHS: |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
350 print "----------------------\nsparsity_update_function" |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
351 theano.printing.debugprint(self.sparsity_update_function) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
352 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
353 def CD_step(self, x): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
354 h1 = self.h_sample_given_x_func(x) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
355 x2 = self.x_sample_given_h_func(h1) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
356 return self.cd_update_function(x, x2) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
357 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
358 def sparsity_step(self, x): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
359 return self.sparsity_update_function(x) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
360 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
361 # these two also operate on minibatches |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
362 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
363 def random_gibbs_samples(self, num_updown_steps): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
364 start_x = self.numpy_rng.rand(*self._get_visibles_shape()) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
365 return self.gibbs_samples_from(start_x, num_updown_steps) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
366 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
367 def gibbs_samples_from(self, start_x, num_updown_steps): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
368 x_sample = start_x |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
369 for i in xrange(num_updown_steps): |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
370 h_sample = self.h_sample_given_x_func(x_sample) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
371 x_sample = self.x_sample_given_h_func(h_sample) |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
372 return x_sample |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
373 |
8d116d4a7593
Added convolutional RBM (ala Lee09) code, imported from my working dir elsewhere. Seems to work for one layer. No subsampling yet.
fsavard
parents:
diff
changeset
|
374 |