Mercurial > ift6266

annotate deep/convolutional_dae/salah_exp/stacked_convolutional_dae_uit.py @ 618:14ba0120baff

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
review response changes
author Yoshua Bengio <bengioy@iro.umontreal.ca>
date Sun, 09 Jan 2011 14:13:23 -0500
parents c05680f8c92f
children
rev   line source
358
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
1 import numpy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
2 import theano
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
3 import time
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
4 import sys
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
5 import theano.tensor as T
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
6 from theano.tensor.shared_randomstreams import RandomStreams
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
7 import theano.sandbox.softsign
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
8 import copy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
9 from theano.tensor.signal import downsample
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
10 from theano.tensor.nnet import conv
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
11
364
c05680f8c92f Fixing a wrong commit and committing more files.
humel
parents: 358
diff changeset
12 sys.path.append('../../')
c05680f8c92f Fixing a wrong commit and committing more files.
humel
parents: 358
diff changeset
13 #import ift6266.datasets
358
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
14 import ift6266.datasets
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
15 from ift6266.baseline.log_reg.log_reg import LogisticRegression
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
16
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
17 from theano.tensor.xlogx import xlogx, xlogy0
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
18 # it's target*log(output)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
19 def binary_cross_entropy(target, output, sum_axis=1):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
20 XE = xlogy0(target, output) + xlogy0((1 - target), (1 - output))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
21 return -T.sum(XE, axis=sum_axis)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
22
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
23
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
24
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
25 class SigmoidalLayer(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
26 def __init__(self, rng, input, n_in, n_out):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
27
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
28 self.input = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
29
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
30 W_values = numpy.asarray( rng.uniform( \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
31 low = -numpy.sqrt(6./(n_in+n_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
32 high = numpy.sqrt(6./(n_in+n_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
33 size = (n_in, n_out)), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
34 self.W = theano.shared(value = W_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
35
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
36 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
37 self.b = theano.shared(value= b_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
38
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
39 self.output = T.tanh(T.dot(input, self.W) + self.b)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
40 self.params = [self.W, self.b]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
41
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
42 class dA_conv(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
43
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
44 def __init__(self, input, filter_shape, corruption_level = 0.1,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
45 shared_W = None, shared_b = None, image_shape = None, num = 0,batch_size=20):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
46
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
47 theano_rng = RandomStreams()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
48
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
49 fan_in = numpy.prod(filter_shape[1:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
50 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
51
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
52 center = theano.shared(value = 1, name="center")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
53 scale = theano.shared(value = 2, name="scale")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
54
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
55 if shared_W != None and shared_b != None :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
56 self.W = shared_W
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
57 self.b = shared_b
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
58 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
59 initial_W = numpy.asarray( numpy.random.uniform(
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
60 low = -numpy.sqrt(6./(fan_in+fan_out)),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
61 high = numpy.sqrt(6./(fan_in+fan_out)),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
62 size = filter_shape), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
63 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
64 self.W = theano.shared(value = initial_W, name = "W")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
65 self.b = theano.shared(value = initial_b, name = "b")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
66
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
67
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
68 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
69
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
70 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
71
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
72 self.x = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
73
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
74 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
75
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
76 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
77 image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
78 unroll_kern=4,unroll_batch=4,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
79 border_mode='valid')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
80
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
81
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
82 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
83
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
84
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
85 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
86 filter_shape[3] ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
87 da_image_shape = [ batch_size, filter_shape[0], image_shape[2]-filter_shape[2]+1,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
88 image_shape[3]-filter_shape[3]+1 ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
89 #import pdb; pdb.set_trace()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
90 initial_W_prime = numpy.asarray( numpy.random.uniform( \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
91 low = -numpy.sqrt(6./(fan_in+fan_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
92 high = numpy.sqrt(6./(fan_in+fan_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
93 size = da_filter_shape), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
94 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
95
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
96 conv2_out = conv.conv2d(self.y, self.W_prime,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
97 filter_shape = da_filter_shape,\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
98 image_shape = da_image_shape, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
99 unroll_kern=4,unroll_batch=4, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
100 border_mode='full')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
101
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
102 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
103
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
104 if num != 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
105 scaled_x = (self.x + center) / scale
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
106 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
107 scaled_x = self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
108 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
109
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
110 self.cost = T.mean(self.L)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
111
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
112 self.params = [ self.W, self.b, self.b_prime ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
113
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
114 class LeNetConvPoolLayer(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
115
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
116 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
117 self.input = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
118
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
119 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
120 self.W = theano.shared(value=W_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
121
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
122 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
123 self.b = theano.shared(value=b_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
124
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
125 conv_out = conv.conv2d(input, self.W,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
126 filter_shape=filter_shape, image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
127 unroll_kern=4,unroll_batch=4)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
128
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
129
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
130 fan_in = numpy.prod(filter_shape[1:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
131 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
132
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
133 W_bound = numpy.sqrt(6./(fan_in + fan_out))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
134 self.W.value = numpy.asarray(
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
135 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
136 dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
137
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
138
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
139 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
140
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
141 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
142 self.params = [self.W, self.b]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
143
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
144
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
145 class CSdA():
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
146 def __init__(self, n_ins_mlp,batch_size, conv_hidden_layers_sizes,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
147 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
148 pretrain_lr, finetune_lr):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
149
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
150 # Just to make sure those are not modified somewhere else afterwards
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
151 hidden_layers_sizes = copy.deepcopy(mlp_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
152 corruption_levels = copy.deepcopy(corruption_levels)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
153
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
154 #update_locals(self, locals())
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
155
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
156
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
157
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
158 self.layers = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
159 self.pretrain_functions = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
160 self.params = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
161 self.n_layers = len(conv_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
162 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
163
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
164 self.x = T.matrix('x') # the data is presented as rasterized images
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
165 self.y = T.ivector('y') # the labels are presented as 1D vector of
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
166
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
167 for i in xrange( self.n_layers ):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
168 filter_shape=conv_hidden_layers_sizes[i][0]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
169 image_shape=conv_hidden_layers_sizes[i][1]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
170 max_poolsize=conv_hidden_layers_sizes[i][2]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
171
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
172 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
173 layer_input=self.x.reshape((batch_size, 1, 32, 32))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
174 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
175 layer_input=self.layers[-1].output
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
176
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
177 layer = LeNetConvPoolLayer(rng, input=layer_input,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
178 image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
179 filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
180 poolsize=max_poolsize)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
181 print 'Convolutional layer', str(i+1), 'created'
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
182
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
183 self.layers += [layer]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
184 self.params += layer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
185
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
186 da_layer = dA_conv(corruption_level = corruption_levels[0],
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
187 input = layer_input,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
188 shared_W = layer.W, shared_b = layer.b,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
189 filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
190 image_shape = image_shape, num=i , batch_size=batch_size)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
191
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
192 gparams = T.grad(da_layer.cost, da_layer.params)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
193
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
194 updates = {}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
195 for param, gparam in zip(da_layer.params, gparams):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
196 updates[param] = param - gparam * pretrain_lr
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
197
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
198 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
199
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
200 self.pretrain_functions += [update_fn]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
201
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
202 for i in xrange( self.mlp_n_layers ):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
203 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
204 input_size = n_ins_mlp
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
205 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
206 input_size = mlp_hidden_layers_sizes[i-1]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
207
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
208 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
209 if len( self.layers ) == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
210 layer_input=self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
211 else :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
212 layer_input = self.layers[-1].output.flatten(2)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
213 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
214 layer_input = self.layers[-1].output
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
215
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
216 layer = SigmoidalLayer(rng, layer_input, input_size,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
217 mlp_hidden_layers_sizes[i] )
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
218
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
219 self.layers += [layer]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
220 self.params += layer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
221
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
222 print 'MLP layer', str(i+1), 'created'
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
223
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
224 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
225 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
226
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
227
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
228 self.params += self.logLayer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
229 self.all_params = self.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
230 cost = self.logLayer.negative_log_likelihood(self.y)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
231
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
232 gparams = T.grad(cost, self.params)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
233
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
234 updates = {}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
235 for param,gparam in zip(self.params, gparams):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
236 updates[param] = param - gparam*finetune_lr
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
237
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
238 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
239
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
240 self.errors = self.logLayer.errors(self.y)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
241
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
242
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
243