Mercurial > ift6266

annotate deep/convolutional_dae/salah_exp/stacked_convolutional_dae_uit.py @ 358:31641a84e0ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial commit for the experimental setup of the denoising convolutional network
author humel
date Thu, 22 Apr 2010 00:49:42 -0400
parents
children c05680f8c92f
rev   line source
358
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
1 import numpy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
2 import theano
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
3 import time
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
4 import sys
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
5 import theano.tensor as T
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
6 from theano.tensor.shared_randomstreams import RandomStreams
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
7 import theano.sandbox.softsign
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
8 import copy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
9 from theano.tensor.signal import downsample
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
10 from theano.tensor.nnet import conv
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
11
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
12
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
13 import ift6266.datasets
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
14 from ift6266.baseline.log_reg.log_reg import LogisticRegression
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
15
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
16 from theano.tensor.xlogx import xlogx, xlogy0
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
17 # it's target*log(output)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
18 def binary_cross_entropy(target, output, sum_axis=1):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
19 XE = xlogy0(target, output) + xlogy0((1 - target), (1 - output))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
20 return -T.sum(XE, axis=sum_axis)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
21
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
22
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
23
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
24 class SigmoidalLayer(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
25 def __init__(self, rng, input, n_in, n_out):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
26
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
27 self.input = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
28
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
29 W_values = numpy.asarray( rng.uniform( \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
30 low = -numpy.sqrt(6./(n_in+n_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
31 high = numpy.sqrt(6./(n_in+n_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
32 size = (n_in, n_out)), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
33 self.W = theano.shared(value = W_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
34
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
35 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
36 self.b = theano.shared(value= b_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
37
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
38 self.output = T.tanh(T.dot(input, self.W) + self.b)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
39 self.params = [self.W, self.b]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
40
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
41 class dA_conv(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
42
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
43 def __init__(self, input, filter_shape, corruption_level = 0.1,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
44 shared_W = None, shared_b = None, image_shape = None, num = 0,batch_size=20):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
45
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
46 theano_rng = RandomStreams()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
47
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
48 fan_in = numpy.prod(filter_shape[1:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
49 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
50
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
51 center = theano.shared(value = 1, name="center")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
52 scale = theano.shared(value = 2, name="scale")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
53
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
54 if shared_W != None and shared_b != None :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
55 self.W = shared_W
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
56 self.b = shared_b
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
57 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
58 initial_W = numpy.asarray( numpy.random.uniform(
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
59 low = -numpy.sqrt(6./(fan_in+fan_out)),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
60 high = numpy.sqrt(6./(fan_in+fan_out)),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
61 size = filter_shape), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
62 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
63 self.W = theano.shared(value = initial_W, name = "W")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
64 self.b = theano.shared(value = initial_b, name = "b")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
65
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
66
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
67 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
68
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
69 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
70
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
71 self.x = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
72
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
73 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
74
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
75 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
76 image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
77 unroll_kern=4,unroll_batch=4,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
78 border_mode='valid')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
79
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
80
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
81 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
82
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
83
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
84 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
85 filter_shape[3] ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
86 da_image_shape = [ batch_size, filter_shape[0], image_shape[2]-filter_shape[2]+1,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
87 image_shape[3]-filter_shape[3]+1 ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
88 #import pdb; pdb.set_trace()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
89 initial_W_prime = numpy.asarray( numpy.random.uniform( \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
90 low = -numpy.sqrt(6./(fan_in+fan_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
91 high = numpy.sqrt(6./(fan_in+fan_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
92 size = da_filter_shape), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
93 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
94
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
95 conv2_out = conv.conv2d(self.y, self.W_prime,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
96 filter_shape = da_filter_shape,\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
97 image_shape = da_image_shape, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
98 unroll_kern=4,unroll_batch=4, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
99 border_mode='full')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
100
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
101 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
102
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
103 if num != 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
104 scaled_x = (self.x + center) / scale
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
105 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
106 scaled_x = self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
107 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
108
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
109 self.cost = T.mean(self.L)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
110
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
111 self.params = [ self.W, self.b, self.b_prime ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
112
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
113 class LeNetConvPoolLayer(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
114
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
115 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
116 self.input = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
117
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
118 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
119 self.W = theano.shared(value=W_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
120
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
121 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
122 self.b = theano.shared(value=b_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
123
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
124 conv_out = conv.conv2d(input, self.W,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
125 filter_shape=filter_shape, image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
126 unroll_kern=4,unroll_batch=4)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
127
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
128
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
129 fan_in = numpy.prod(filter_shape[1:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
130 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
131
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
132 W_bound = numpy.sqrt(6./(fan_in + fan_out))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
133 self.W.value = numpy.asarray(
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
134 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
135 dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
136
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
137
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
138 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
139
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
140 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
141 self.params = [self.W, self.b]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
142
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
143
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
144 class CSdA():
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
145 def __init__(self, n_ins_mlp,batch_size, conv_hidden_layers_sizes,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
146 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
147 pretrain_lr, finetune_lr):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
148
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
149 # Just to make sure those are not modified somewhere else afterwards
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
150 hidden_layers_sizes = copy.deepcopy(mlp_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
151 corruption_levels = copy.deepcopy(corruption_levels)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
152
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
153 #update_locals(self, locals())
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
154
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
155
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
156
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
157 self.layers = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
158 self.pretrain_functions = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
159 self.params = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
160 self.n_layers = len(conv_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
161 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
162
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
163 self.x = T.matrix('x') # the data is presented as rasterized images
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
164 self.y = T.ivector('y') # the labels are presented as 1D vector of
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
165
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
166 for i in xrange( self.n_layers ):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
167 filter_shape=conv_hidden_layers_sizes[i][0]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
168 image_shape=conv_hidden_layers_sizes[i][1]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
169 max_poolsize=conv_hidden_layers_sizes[i][2]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
170
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
171 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
172 layer_input=self.x.reshape((batch_size, 1, 32, 32))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
173 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
174 layer_input=self.layers[-1].output
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
175
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
176 layer = LeNetConvPoolLayer(rng, input=layer_input,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
177 image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
178 filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
179 poolsize=max_poolsize)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
180 print 'Convolutional layer', str(i+1), 'created'
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
181
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
182 self.layers += [layer]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
183 self.params += layer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
184
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
185 da_layer = dA_conv(corruption_level = corruption_levels[0],
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
186 input = layer_input,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
187 shared_W = layer.W, shared_b = layer.b,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
188 filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
189 image_shape = image_shape, num=i , batch_size=batch_size)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
190
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
191 gparams = T.grad(da_layer.cost, da_layer.params)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
192
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
193 updates = {}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
194 for param, gparam in zip(da_layer.params, gparams):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
195 updates[param] = param - gparam * pretrain_lr
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
196
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
197 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
198
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
199 self.pretrain_functions += [update_fn]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
200
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
201 for i in xrange( self.mlp_n_layers ):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
202 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
203 input_size = n_ins_mlp
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
204 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
205 input_size = mlp_hidden_layers_sizes[i-1]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
206
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
207 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
208 if len( self.layers ) == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
209 layer_input=self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
210 else :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
211 layer_input = self.layers[-1].output.flatten(2)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
212 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
213 layer_input = self.layers[-1].output
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
214
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
215 layer = SigmoidalLayer(rng, layer_input, input_size,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
216 mlp_hidden_layers_sizes[i] )
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
217
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
218 self.layers += [layer]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
219 self.params += layer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
220
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
221 print 'MLP layer', str(i+1), 'created'
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
222
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
223 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
224 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
225
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
226
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
227 self.params += self.logLayer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
228 self.all_params = self.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
229 cost = self.logLayer.negative_log_likelihood(self.y)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
230
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
231 gparams = T.grad(cost, self.params)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
232
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
233 updates = {}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
234 for param,gparam in zip(self.params, gparams):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
235 updates[param] = param - gparam*finetune_lr
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
236
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
237 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
238
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
239 self.errors = self.logLayer.errors(self.y)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
240
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
241
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
242