annotate deep/rbm/rbm.py @ 422:e7790db265b1

Basic text for section 3, add a bit more detail to section 4.2.2
author Arnaud Bergeron <abergeron@gmail.com>
date Fri, 30 Apr 2010 16:24:30 -0400
parents e36ccffb3870
children
rev   line source
347
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
1 """This tutorial introduces restricted boltzmann machines (RBM) using Theano.
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
2
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
3 Boltzmann Machines (BMs) are a particular form of energy-based model which
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
4 contain hidden variables. Restricted Boltzmann Machines further restrict BMs
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
5 to those without visible-visible and hidden-hidden connections.
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
6 """
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
7
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
8 import numpy, time, cPickle, gzip, PIL.Image
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
9
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
10 import theano
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
11 import theano.tensor as T
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
12 import os
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
13 import pdb
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
14 import numpy
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
15 import pylab
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
16 import time
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
17 import theano.tensor.nnet
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
18 import pylearn
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
19 #import ift6266
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
20 import theano,pylearn.version #,ift6266
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
21 from pylearn.io import filetensor as ft
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
22 #from ift6266 import datasets
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
23
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
24 from jobman.tools import DD, flatten
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
25 from jobman import sql
347
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
26
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
27 from theano.tensor.shared_randomstreams import RandomStreams
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
28
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
29 from utils import tile_raster_images
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
30 from logistic_sgd import load_data
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
31
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
32 class RBM(object):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
33 """Restricted Boltzmann Machine (RBM) """
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
34 def __init__(self, input=None, n_visible=32*32, n_hidden=500, \
347
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
35 W = None, hbias = None, vbias = None, numpy_rng = None,
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
36 theano_rng = None):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
37 """
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
38 RBM constructor. Defines the parameters of the model along with
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
39 basic operations for inferring hidden from visible (and vice-versa),
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
40 as well as for performing CD updates.
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
41
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
42 :param input: None for standalone RBMs or symbolic variable if RBM is
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
43 part of a larger graph.
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
44
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
45 :param n_visible: number of visible units
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
46
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
47 :param n_hidden: number of hidden units
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
48
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
49 :param W: None for standalone RBMs or symbolic variable pointing to a
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
50 shared weight matrix in case RBM is part of a DBN network; in a DBN,
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
51 the weights are shared between RBMs and layers of a MLP
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
52
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
53 :param hbias: None for standalone RBMs or symbolic variable pointing
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
54 to a shared hidden units bias vector in case RBM is part of a
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
55 different network
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
56
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
57 :param vbias: None for standalone RBMs or a symbolic variable
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
58 pointing to a shared visible units bias
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
59 """
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
60
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
61 self.n_visible = n_visible
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
62 self.n_hidden = n_hidden
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
63
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
64
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
65 if W is None :
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
66 # W is initialized with `initial_W` which is uniformely sampled
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
67 # from -6./sqrt(n_visible+n_hidden) and 6./sqrt(n_hidden+n_visible)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
68 # the output of uniform if converted using asarray to dtype
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
69 # theano.config.floatX so that the code is runable on GPU
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
70 initial_W = numpy.asarray( numpy.random.uniform(
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
71 low = -numpy.sqrt(6./(n_hidden+n_visible)),
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
72 high = numpy.sqrt(6./(n_hidden+n_visible)),
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
73 size = (n_visible, n_hidden)),
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
74 dtype = theano.config.floatX)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
75 # theano shared variables for weights and biases
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
76 W = theano.shared(value = initial_W, name = 'W')
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
77
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
78 if hbias is None :
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
79 # create shared variable for hidden units bias
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
80 hbias = theano.shared(value = numpy.zeros(n_hidden,
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
81 dtype = theano.config.floatX), name='hbias')
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
82
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
83 if vbias is None :
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
84 # create shared variable for visible units bias
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
85 vbias = theano.shared(value =numpy.zeros(n_visible,
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
86 dtype = theano.config.floatX),name='vbias')
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
87
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
88 if numpy_rng is None:
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
89 # create a number generator
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
90 numpy_rng = numpy.random.RandomState(1234)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
91
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
92 if theano_rng is None :
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
93 theano_rng = RandomStreams(numpy_rng.randint(2**30))
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
94
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
95
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
96 # initialize input layer for standalone RBM or layer0 of DBN
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
97 self.input = input if input else T.dmatrix('input')
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
98
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
99 self.W = W
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
100 self.hbias = hbias
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
101 self.vbias = vbias
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
102 self.theano_rng = theano_rng
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
103
347
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
104 # **** WARNING: It is not a good idea to put things in this list
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
105 # other than shared variables created in this function.
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
106 self.params = [self.W, self.hbias, self.vbias]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
107 self.batch_size = self.input.shape[0]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
108
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
109 def free_energy(self, v_sample):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
110 ''' Function to compute the free energy '''
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
111 wx_b = T.dot(v_sample, self.W) + self.hbias
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
112 vbias_term = T.sum(T.dot(v_sample, self.vbias))
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
113 hidden_term = T.sum(T.log(1+T.exp(wx_b)))
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
114 return -hidden_term - vbias_term
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
115
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
116 def sample_h_given_v(self, v0_sample):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
117 ''' This function infers state of hidden units given visible units '''
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
118 # compute the activation of the hidden units given a sample of the visibles
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
119 h1_mean = T.nnet.sigmoid(T.dot(v0_sample, self.W) + self.hbias)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
120 # get a sample of the hiddens given their activation
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
121 h1_sample = self.theano_rng.binomial(size = h1_mean.shape, n = 1, prob = h1_mean)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
122 return [h1_mean, h1_sample]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
123
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
124 def sample_v_given_h(self, h0_sample):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
125 ''' This function infers state of visible units given hidden units '''
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
126 # compute the activation of the visible given the hidden sample
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
127 v1_mean = T.nnet.sigmoid(T.dot(h0_sample, self.W.T) + self.vbias)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
128 # get a sample of the visible given their activation
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
129 v1_sample = self.theano_rng.binomial(size = v1_mean.shape,n = 1,prob = v1_mean)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
130 return [v1_mean, v1_sample]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
131
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
132 def gibbs_hvh(self, h0_sample):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
133 ''' This function implements one step of Gibbs sampling,
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
134 starting from the hidden state'''
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
135 v1_mean, v1_sample = self.sample_v_given_h(h0_sample)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
136 h1_mean, h1_sample = self.sample_h_given_v(v1_sample)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
137 return [v1_mean, v1_sample, h1_mean, h1_sample]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
138
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
139 def gibbs_vhv(self, v0_sample):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
140 ''' This function implements one step of Gibbs sampling,
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
141 starting from the visible state'''
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
142 h1_mean, h1_sample = self.sample_h_given_v(v0_sample)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
143 v1_mean, v1_sample = self.sample_v_given_h(h1_sample)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
144 return [h1_mean, h1_sample, v1_mean, v1_sample]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
145
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
146 def cd(self, lr = 0.1, persistent=None, k=1):
347
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
147 """
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
148 This functions implements one step of CD-1 or PCD-1
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
149
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
150 :param lr: learning rate used to train the RBM
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
151 :param persistent: None for CD. For PCD, shared variable containing old state
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
152 of Gibbs chain. This must be a shared variable of size (batch size, number of
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
153 hidden units).
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
154
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
155 Returns the updates dictionary. The dictionary contains the update rules for weights
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
156 and biases but also an update of the shared variable used to store the persistent
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
157 chain, if one is used.
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
158 """
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
159
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
160 # compute positive phase
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
161 ph_mean, ph_sample = self.sample_h_given_v(self.input)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
162
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
163 # decide how to initialize persistent chain:
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
164 # for CD, we use the newly generate hidden sample
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
165 # for PCD, we initialize from the old state of the chain
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
166 if persistent is None:
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
167 chain_start = ph_sample
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
168 else:
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
169 chain_start = persistent
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
170
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
171 # perform actual negative phase (the CD-1)
347
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
172 [nv_mean, nv_sample, nh_mean, nh_sample] = self.gibbs_hvh(chain_start)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
173
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
174 #perform CD-k
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
175 if k-1>0:
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
176 for i in range(k-1):
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
177 [nv_mean, nv_sample, nh_mean, nh_sample] = self.gibbs_hvh(nh_sample)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
178
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
179
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
180
347
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
181 # determine gradients on RBM parameters
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
182 g_vbias = T.sum( self.input - nv_mean, axis = 0)/self.batch_size
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
183 g_hbias = T.sum( ph_mean - nh_mean, axis = 0)/self.batch_size
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
184 g_W = T.dot(ph_mean.T, self.input )/ self.batch_size - \
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
185 T.dot(nh_mean.T, nv_mean )/ self.batch_size
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
186
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
187 gparams = [g_W.T, g_hbias, g_vbias]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
188
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
189 # constructs the update dictionary
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
190 updates = {}
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
191 for gparam, param in zip(gparams, self.params):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
192 updates[param] = param + gparam * lr
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
193
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
194 if persistent:
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
195 # Note that this works only if persistent is a shared variable
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
196 updates[persistent] = T.cast(nh_sample, dtype=theano.config.floatX)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
197 # pseudo-likelihood is a better proxy for PCD
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
198 cost = self.get_pseudo_likelihood_cost(updates)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
199 else:
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
200 # reconstruction cross-entropy is a better proxy for CD
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
201 cost = self.get_reconstruction_cost(updates, nv_mean)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
202
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
203 return cost, updates
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
204
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
205 def get_pseudo_likelihood_cost(self, updates):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
206 """Stochastic approximation to the pseudo-likelihood"""
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
207
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
208 # index of bit i in expression p(x_i | x_{\i})
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
209 bit_i_idx = theano.shared(value=0, name = 'bit_i_idx')
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
210
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
211 # binarize the input image by rounding to nearest integer
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
212 xi = T.iround(self.input)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
213
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
214 # calculate free energy for the given bit configuration
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
215 fe_xi = self.free_energy(xi)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
216
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
217 # flip bit x_i of matrix xi and preserve all other bits x_{\i}
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
218 # Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
219 # NB: slice(start,stop,step) is the python object used for
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
220 # slicing, e.g. to index matrix x as follows: x[start:stop:step]
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
221 xi_flip = T.setsubtensor(xi, 1-xi[:, bit_i_idx],
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
222 idx_list=(slice(None,None,None),bit_i_idx))
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
223
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
224 # calculate free energy with bit flipped
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
225 fe_xi_flip = self.free_energy(xi_flip)
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
226
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
227 # equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\i})))
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
228 cost = self.n_visible * T.log(T.nnet.sigmoid(fe_xi_flip - fe_xi))
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
229
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
230 # increment bit_i_idx % number as part of updates
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
231 updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
232
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
233 return cost
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
234
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
235 def get_reconstruction_cost(self, updates, nv_mean):
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
236 """Approximation to the reconstruction error"""
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
237
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
238 cross_entropy = T.mean(
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
239 T.sum(self.input*T.log(nv_mean) +
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
240 (1 - self.input)*T.log(1-nv_mean), axis = 1))
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
241
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
242 return cross_entropy
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
243
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
244
9685e9d94cc4 base class for an rbm
goldfinger
parents:
diff changeset
245
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
246 def test_rbm(b_size = 20, nhidden = 1000, kk = 1, persistance = 0):
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
247 """
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
248 Demonstrate ***
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
249
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
250 This is demonstrated on MNIST.
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
251
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
252 :param learning_rate: learning rate used for training the RBM
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
253
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
254 :param training_epochs: number of epochs used for training
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
255
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
256 :param dataset: path the the pickled dataset
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
257
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
258 """
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
259
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
260 learning_rate=0.1
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
261
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
262 # if data_set==0:
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
263 # datasets=datasets.nist_all()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
264 # elif data_set==1:
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
265 # datasets=datasets.nist_P07()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
266 # elif data_set==2:
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
267 # datasets=datasets.PNIST07()
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
268
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
269
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
270 data_path = '/data/lisa/data/nist/by_class/'
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
271 f = open(data_path+'all/all_train_data.ft')
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
272 g = open(data_path+'all/all_train_labels.ft')
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
273 h = open(data_path+'all/all_test_data.ft')
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
274 i = open(data_path+'all/all_test_labels.ft')
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
275
375
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
276 train_set_x_uint8 = theano.shared(ft.read(f))
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
277 test_set_x_uint8 = theano.shared(ft.read(h))
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
278
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
279
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
280 train_set_x = T.cast(train_set_x_uint8/255.,theano.config.floatX)
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
281 train_set_y = ft.read(g)
375
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
282 test_set_x = T.cast(test_set_x_uint8/255.,theano.config.floatX)
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
283 test_set_y = ft.read(i)
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
284
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
285 f.close()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
286 g.close()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
287 i.close()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
288 h.close()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
289
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
290 #t = len(train_set_x)
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
291
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
292 # revoir la recuperation des donnees
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
293 ## dataset = load_data(dataset)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
294 ##
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
295 ## train_set_x, train_set_y = datasets[0]
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
296 ## test_set_x , test_set_y = datasets[2]
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
297 training_epochs = 1 # a determiner
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
298
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
299 batch_size = b_size # size of the minibatch
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
300
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
301 # compute number of minibatches for training, validation and testing
375
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
302 n_train_batches = train_set_x_uint8.value.shape[0] / batch_size
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
303
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
304 # allocate symbolic variables for the data
375
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
305 index = T.lscalar() # index to a [mini]batch
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
306 x = T.matrix('x') # the data is presented as rasterized images
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
307
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
308 rng = numpy.random.RandomState(123)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
309 theano_rng = RandomStreams( rng.randint(2**30))
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
310
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
311
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
312 # construct the RBM class
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
313 rbm = RBM( input = x, n_visible=32*32, \
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
314 n_hidden = nhidden, numpy_rng = rng, theano_rng = theano_rng)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
315
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
316
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
317 # initialize storage fot the persistent chain (state = hidden layer of chain)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
318 if persistance == 1:
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
319 persistent_chain = theano.shared(numpy.zeros((batch_size, 500)))
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
320 # get the cost and the gradient corresponding to one step of CD
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
321 cost, updates = rbm.cd(lr=learning_rate, persistent=persistent_chain, k= kk)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
322
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
323 else:
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
324 # get the cost and the gradient corresponding to one step of CD
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
325 #persistance_chain = None
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
326 cost, updates = rbm.cd(lr=learning_rate, persistent=None, k= kk)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
327
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
328 #################################
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
329 # Training the RBM #
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
330 #################################
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
331 #os.chdir('~')
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
332 dirname = str(persistance) + '_' + str(nhidden) + '_' + str(b_size) + '_'+ str(kk)
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
333 os.makedirs(dirname)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
334 os.chdir(dirname)
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
335 print 'yes'
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
336 # it is ok for a theano function to have no output
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
337 # the purpose of train_rbm is solely to update the RBM parameters
375
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
338 print type(batch_size)
e36ccffb3870 Changes to cast NIST data to floatX for rbm code
Xavier Glorot <glorotxa@iro.umontreal.ca>
parents: 372
diff changeset
339 print index.dtype
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
340 train_rbm = theano.function([index], cost,
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
341 updates = updates,
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
342 givens = { x: train_set_x[index*batch_size:(index+1)*batch_size]})
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
343
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
344 print 'yep'
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
345 plotting_time = 0.0
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
346 start_time = time.clock()
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
347 bufsize = 1000
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
348
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
349 # go through training epochs
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
350 costs = []
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
351 for epoch in xrange(training_epochs):
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
352
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
353 # go through the training set
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
354 mean_cost = []
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
355 for batch_index in xrange(n_train_batches):
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
356 mean_cost += [train_rbm(batch_index)]
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
357 # for mini_x, mini_y in datasets.train(b_size):
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
358 # mean_cost += [train_rbm(mini_x)]
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
359 ## learning_rate = learning_rate - 0.0001
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
360 ## learning_rate = learning_rate/(tau+( epoch*batch_index*batch_size))
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
361
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
362 #learning_rate = learning_rate/10
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
363
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
364 costs.append(numpy.mean(mean_cost))
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
365
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
366 # Plot filters after each training epoch
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
367 plotting_start = time.clock()
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
368 # Construct image from the weight matrix
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
369 image = PIL.Image.fromarray(tile_raster_images( X = rbm.W.value.T,
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
370 img_shape = (32,32),tile_shape = (10,10),
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
371 tile_spacing=(1,1)))
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
372 image.save('filters_at_epoch_%i.png'%epoch)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
373 plotting_stop = time.clock()
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
374 plotting_time += (plotting_stop - plotting_start)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
375
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
376 end_time = time.clock()
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
377
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
378 pretraining_time = (end_time - start_time) - plotting_time
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
379
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
380
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
381
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
382 #################################
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
383 # Sampling from the RBM #
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
384 #################################
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
385
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
386 # find out the number of test samples
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
387 #number_of_test_samples = 100
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
388 number_of_test_samples = test_set_x.value.shape[0]
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
389
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
390 #test_set_x, test_y = datasets.test(100*b_size)
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
391 # pick random test examples, with which to initialize the persistent chain
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
392 test_idx = rng.randint(number_of_test_samples - b_size)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
393 persistent_vis_chain = theano.shared(test_set_x.value[test_idx:test_idx+b_size])
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
394
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
395 # define one step of Gibbs sampling (mf = mean-field)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
396 [hid_mf, hid_sample, vis_mf, vis_sample] = rbm.gibbs_vhv(persistent_vis_chain)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
397
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
398 # the sample at the end of the channel is returned by ``gibbs_1`` as
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
399 # its second output; note that this is computed as a binomial draw,
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
400 # therefore it is formed of ints (0 and 1) and therefore needs to
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
401 # be converted to the same dtype as ``persistent_vis_chain``
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
402 vis_sample = T.cast(vis_sample, dtype=theano.config.floatX)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
403
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
404 # construct the function that implements our persistent chain
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
405 # we generate the "mean field" activations for plotting and the actual samples for
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
406 # reinitializing the state of our persistent chain
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
407 sample_fn = theano.function([], [vis_mf, vis_sample],
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
408 updates = { persistent_vis_chain:vis_sample})
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
409
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
410 # sample the RBM, plotting every `plot_every`-th sample; do this
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
411 # until you plot at least `n_samples`
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
412 n_samples = 10
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
413 # run minibatch size chains for gibbs samples (number of negative particles)
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
414 plot_every = b_size
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
415
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
416 for idx in xrange(n_samples):
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
417
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
418 # do `plot_every` intermediate samplings of which we do not care
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
419 for jdx in xrange(plot_every):
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
420 vis_mf, vis_sample = sample_fn()
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
421
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
422 # construct image
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
423 image = PIL.Image.fromarray(tile_raster_images(
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
424 X = vis_mf,
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
425 img_shape = (32,32),
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
426 tile_shape = (10,10),
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
427 tile_spacing = (1,1) ) )
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
428 #print ' ... plotting sample ', idx
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
429 image.save('sample_%i_step_%i.png'%(idx,idx*jdx))
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
430
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
431 #save the model
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
432 model = [rbm.W, rbm.vbias, rbm.hbias]
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
433 f = fopen('params.txt', 'w')
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
434 cPickle.dump(model, f, protocol = -1)
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
435 f.close()
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
436 #os.chdir('./..')
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
437 return numpy.mean(costs), pretraining_time*36
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
438
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
439
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
440 def experiment(state, channel):
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
441
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
442 (mean_cost, time_execution) = test_rbm(b_size = state.b_size,\
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
443 nhidden = state.ndidden,\
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
444 kk = state.kk,\
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
445 persistance = state.persistance,\
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
446 )
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
447
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
448 state.mean_costs = mean_costs
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
449 state.time_execution = time_execution
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
450 pylearn.version.record_versions(state,[theano,ift6266,pylearn])
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
451 return channel.COMPLETE
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
452
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
453 if __name__ == '__main__':
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
454
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
455 TABLE_NAME='RBM_tapha'
369
d81284e13d77 modified to run experiments with PNIST
goldfinger
parents: 347
diff changeset
456
372
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
457 # DB path...
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
458 test_rbm()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
459 #db = sql.db('postgres://ift6266h10:f0572cd63b@gershwin/ift6266h10_db/'+ TABLE_NAME)
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
460
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
461 #state = DD()
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
462 #for b_size in 50, 75, 100:
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
463 # state.b_size = b_size
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
464 # for nhidden in 1000,1250,1500:
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
465 # state.nhidden = nhidden
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
466 # for kk in 1,2,3,4:
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
467 # state.kk = kk
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
468 # for persistance in 0,1:
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
469 # state.persistance = persistance
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
470 # sql.insert_job(rbm.experiment, flatten(state), db)
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
471
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
472
1e99dc965b5b correcting some bugs
goldfinger
parents: 369
diff changeset
473 #db.createView(TABLE_NAME + 'view')