Mercurial > ift6266

comparison deep/convolutional_dae/salah_exp/stacked_convolutional_dae_uit.py @ 358:31641a84e0ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial commit for the experimental setup of the denoising convolutional network
author humel
date Thu, 22 Apr 2010 00:49:42 -0400
parents
children c05680f8c92f
comparison
equal deleted inserted replaced
357:9a7b74927f7d 358:31641a84e0ae
1 import numpy
2 import theano
3 import time
4 import sys
5 import theano.tensor as T
6 from theano.tensor.shared_randomstreams import RandomStreams
7 import theano.sandbox.softsign
8 import copy
9 from theano.tensor.signal import downsample
10 from theano.tensor.nnet import conv
11
12
13 import ift6266.datasets
14 from ift6266.baseline.log_reg.log_reg import LogisticRegression
15
16 from theano.tensor.xlogx import xlogx, xlogy0
17 # it's target*log(output)
18 def binary_cross_entropy(target, output, sum_axis=1):
19 XE = xlogy0(target, output) + xlogy0((1 - target), (1 - output))
20 return -T.sum(XE, axis=sum_axis)
21
22
23
24 class SigmoidalLayer(object):
25 def __init__(self, rng, input, n_in, n_out):
26
27 self.input = input
28
29 W_values = numpy.asarray( rng.uniform( \
30 low = -numpy.sqrt(6./(n_in+n_out)), \
31 high = numpy.sqrt(6./(n_in+n_out)), \
32 size = (n_in, n_out)), dtype = theano.config.floatX)
33 self.W = theano.shared(value = W_values)
34
35 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
36 self.b = theano.shared(value= b_values)
37
38 self.output = T.tanh(T.dot(input, self.W) + self.b)
39 self.params = [self.W, self.b]
40
41 class dA_conv(object):
42
43 def __init__(self, input, filter_shape, corruption_level = 0.1,
44 shared_W = None, shared_b = None, image_shape = None, num = 0,batch_size=20):
45
46 theano_rng = RandomStreams()
47
48 fan_in = numpy.prod(filter_shape[1:])
49 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
50
51 center = theano.shared(value = 1, name="center")
52 scale = theano.shared(value = 2, name="scale")
53
54 if shared_W != None and shared_b != None :
55 self.W = shared_W
56 self.b = shared_b
57 else:
58 initial_W = numpy.asarray( numpy.random.uniform(
59 low = -numpy.sqrt(6./(fan_in+fan_out)),
60 high = numpy.sqrt(6./(fan_in+fan_out)),
61 size = filter_shape), dtype = theano.config.floatX)
62 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
63 self.W = theano.shared(value = initial_W, name = "W")
64 self.b = theano.shared(value = initial_b, name = "b")
65
66
67 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
68
69 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
70
71 self.x = input
72
73 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
74
75 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
76 image_shape=image_shape,
77 unroll_kern=4,unroll_batch=4,
78 border_mode='valid')
79
80
81 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
82
83
84 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
85 filter_shape[3] ]
86 da_image_shape = [ batch_size, filter_shape[0], image_shape[2]-filter_shape[2]+1,
87 image_shape[3]-filter_shape[3]+1 ]
88 #import pdb; pdb.set_trace()
89 initial_W_prime = numpy.asarray( numpy.random.uniform( \
90 low = -numpy.sqrt(6./(fan_in+fan_out)), \
91 high = numpy.sqrt(6./(fan_in+fan_out)), \
92 size = da_filter_shape), dtype = theano.config.floatX)
93 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
94
95 conv2_out = conv.conv2d(self.y, self.W_prime,
96 filter_shape = da_filter_shape,\
97 image_shape = da_image_shape, \
98 unroll_kern=4,unroll_batch=4, \
99 border_mode='full')
100
101 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
102
103 if num != 0 :
104 scaled_x = (self.x + center) / scale
105 else:
106 scaled_x = self.x
107 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
108
109 self.cost = T.mean(self.L)
110
111 self.params = [ self.W, self.b, self.b_prime ]
112
113 class LeNetConvPoolLayer(object):
114
115 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
116 self.input = input
117
118 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
119 self.W = theano.shared(value=W_values)
120
121 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
122 self.b = theano.shared(value=b_values)
123
124 conv_out = conv.conv2d(input, self.W,
125 filter_shape=filter_shape, image_shape=image_shape,
126 unroll_kern=4,unroll_batch=4)
127
128
129 fan_in = numpy.prod(filter_shape[1:])
130 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
131
132 W_bound = numpy.sqrt(6./(fan_in + fan_out))
133 self.W.value = numpy.asarray(
134 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
135 dtype = theano.config.floatX)
136
137
138 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
139
140 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
141 self.params = [self.W, self.b]
142
143
144 class CSdA():
145 def __init__(self, n_ins_mlp,batch_size, conv_hidden_layers_sizes,
146 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
147 pretrain_lr, finetune_lr):
148
149 # Just to make sure those are not modified somewhere else afterwards
150 hidden_layers_sizes = copy.deepcopy(mlp_hidden_layers_sizes)
151 corruption_levels = copy.deepcopy(corruption_levels)
152
153 #update_locals(self, locals())
154
155
156
157 self.layers = []
158 self.pretrain_functions = []
159 self.params = []
160 self.n_layers = len(conv_hidden_layers_sizes)
161 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
162
163 self.x = T.matrix('x') # the data is presented as rasterized images
164 self.y = T.ivector('y') # the labels are presented as 1D vector of
165
166 for i in xrange( self.n_layers ):
167 filter_shape=conv_hidden_layers_sizes[i][0]
168 image_shape=conv_hidden_layers_sizes[i][1]
169 max_poolsize=conv_hidden_layers_sizes[i][2]
170
171 if i == 0 :
172 layer_input=self.x.reshape((batch_size, 1, 32, 32))
173 else:
174 layer_input=self.layers[-1].output
175
176 layer = LeNetConvPoolLayer(rng, input=layer_input,
177 image_shape=image_shape,
178 filter_shape=filter_shape,
179 poolsize=max_poolsize)
180 print 'Convolutional layer', str(i+1), 'created'
181
182 self.layers += [layer]
183 self.params += layer.params
184
185 da_layer = dA_conv(corruption_level = corruption_levels[0],
186 input = layer_input,
187 shared_W = layer.W, shared_b = layer.b,
188 filter_shape=filter_shape,
189 image_shape = image_shape, num=i , batch_size=batch_size)
190
191 gparams = T.grad(da_layer.cost, da_layer.params)
192
193 updates = {}
194 for param, gparam in zip(da_layer.params, gparams):
195 updates[param] = param - gparam * pretrain_lr
196
197 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
198
199 self.pretrain_functions += [update_fn]
200
201 for i in xrange( self.mlp_n_layers ):
202 if i == 0 :
203 input_size = n_ins_mlp
204 else:
205 input_size = mlp_hidden_layers_sizes[i-1]
206
207 if i == 0 :
208 if len( self.layers ) == 0 :
209 layer_input=self.x
210 else :
211 layer_input = self.layers[-1].output.flatten(2)
212 else:
213 layer_input = self.layers[-1].output
214
215 layer = SigmoidalLayer(rng, layer_input, input_size,
216 mlp_hidden_layers_sizes[i] )
217
218 self.layers += [layer]
219 self.params += layer.params
220
221 print 'MLP layer', str(i+1), 'created'
222
223 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
224 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
225
226
227 self.params += self.logLayer.params
228 self.all_params = self.params
229 cost = self.logLayer.negative_log_likelihood(self.y)
230
231 gparams = T.grad(cost, self.params)
232
233 updates = {}
234 for param,gparam in zip(self.params, gparams):
235 updates[param] = param - gparam*finetune_lr
236
237 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
238
239 self.errors = self.logLayer.errors(self.y)
240
241
242