Mercurial > ift6266

annotate deep/convolutional_dae/salah_exp/stacked_convolutional_dae_uit.py @ 610:9e08366652d6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added poster for NIPS2010 (openoffice draw source file)
author fsavard
date Thu, 02 Dec 2010 17:10:06 -0500
parents c05680f8c92f
children
rev   line source
358
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
1 import numpy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
2 import theano
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
3 import time
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
4 import sys
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
5 import theano.tensor as T
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
6 from theano.tensor.shared_randomstreams import RandomStreams
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
7 import theano.sandbox.softsign
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
8 import copy
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
9 from theano.tensor.signal import downsample
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
10 from theano.tensor.nnet import conv
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
11
364
c05680f8c92f Fixing a wrong commit and committing more files.
humel
parents: 358
diff changeset
12 sys.path.append('../../')
c05680f8c92f Fixing a wrong commit and committing more files.
humel
parents: 358
diff changeset
13 #import ift6266.datasets
358
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
14 import ift6266.datasets
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
15 from ift6266.baseline.log_reg.log_reg import LogisticRegression
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
16
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
17 from theano.tensor.xlogx import xlogx, xlogy0
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
18 # it's target*log(output)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
19 def binary_cross_entropy(target, output, sum_axis=1):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
20 XE = xlogy0(target, output) + xlogy0((1 - target), (1 - output))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
21 return -T.sum(XE, axis=sum_axis)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
22
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
23
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
24
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
25 class SigmoidalLayer(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
26 def __init__(self, rng, input, n_in, n_out):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
27
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
28 self.input = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
29
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
30 W_values = numpy.asarray( rng.uniform( \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
31 low = -numpy.sqrt(6./(n_in+n_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
32 high = numpy.sqrt(6./(n_in+n_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
33 size = (n_in, n_out)), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
34 self.W = theano.shared(value = W_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
35
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
36 b_values = numpy.zeros((n_out,), dtype= theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
37 self.b = theano.shared(value= b_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
38
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
39 self.output = T.tanh(T.dot(input, self.W) + self.b)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
40 self.params = [self.W, self.b]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
41
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
42 class dA_conv(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
43
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
44 def __init__(self, input, filter_shape, corruption_level = 0.1,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
45 shared_W = None, shared_b = None, image_shape = None, num = 0,batch_size=20):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
46
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
47 theano_rng = RandomStreams()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
48
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
49 fan_in = numpy.prod(filter_shape[1:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
50 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
51
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
52 center = theano.shared(value = 1, name="center")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
53 scale = theano.shared(value = 2, name="scale")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
54
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
55 if shared_W != None and shared_b != None :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
56 self.W = shared_W
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
57 self.b = shared_b
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
58 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
59 initial_W = numpy.asarray( numpy.random.uniform(
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
60 low = -numpy.sqrt(6./(fan_in+fan_out)),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
61 high = numpy.sqrt(6./(fan_in+fan_out)),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
62 size = filter_shape), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
63 initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
64 self.W = theano.shared(value = initial_W, name = "W")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
65 self.b = theano.shared(value = initial_b, name = "b")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
66
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
67
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
68 initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
69
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
70 self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
71
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
72 self.x = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
73
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
74 self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
75
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
76 conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
77 image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
78 unroll_kern=4,unroll_batch=4,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
79 border_mode='valid')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
80
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
81
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
82 self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
83
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
84
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
85 da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2],\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
86 filter_shape[3] ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
87 da_image_shape = [ batch_size, filter_shape[0], image_shape[2]-filter_shape[2]+1,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
88 image_shape[3]-filter_shape[3]+1 ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
89 #import pdb; pdb.set_trace()
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
90 initial_W_prime = numpy.asarray( numpy.random.uniform( \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
91 low = -numpy.sqrt(6./(fan_in+fan_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
92 high = numpy.sqrt(6./(fan_in+fan_out)), \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
93 size = da_filter_shape), dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
94 self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime")
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
95
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
96 conv2_out = conv.conv2d(self.y, self.W_prime,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
97 filter_shape = da_filter_shape,\
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
98 image_shape = da_image_shape, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
99 unroll_kern=4,unroll_batch=4, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
100 border_mode='full')
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
101
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
102 self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
103
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
104 if num != 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
105 scaled_x = (self.x + center) / scale
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
106 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
107 scaled_x = self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
108 self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 )
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
109
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
110 self.cost = T.mean(self.L)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
111
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
112 self.params = [ self.W, self.b, self.b_prime ]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
113
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
114 class LeNetConvPoolLayer(object):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
115
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
116 def __init__(self, rng, input, filter_shape, image_shape=None, poolsize=(2,2)):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
117 self.input = input
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
118
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
119 W_values = numpy.zeros(filter_shape, dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
120 self.W = theano.shared(value=W_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
121
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
122 b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
123 self.b = theano.shared(value=b_values)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
124
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
125 conv_out = conv.conv2d(input, self.W,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
126 filter_shape=filter_shape, image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
127 unroll_kern=4,unroll_batch=4)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
128
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
129
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
130 fan_in = numpy.prod(filter_shape[1:])
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
131 fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
132
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
133 W_bound = numpy.sqrt(6./(fan_in + fan_out))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
134 self.W.value = numpy.asarray(
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
135 rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
136 dtype = theano.config.floatX)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
137
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
138
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
139 pooled_out = downsample.max_pool2D(conv_out, poolsize, ignore_border=True)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
140
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
141 self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
142 self.params = [self.W, self.b]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
143
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
144
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
145 class CSdA():
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
146 def __init__(self, n_ins_mlp,batch_size, conv_hidden_layers_sizes,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
147 mlp_hidden_layers_sizes, corruption_levels, rng, n_out,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
148 pretrain_lr, finetune_lr):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
149
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
150 # Just to make sure those are not modified somewhere else afterwards
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
151 hidden_layers_sizes = copy.deepcopy(mlp_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
152 corruption_levels = copy.deepcopy(corruption_levels)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
153
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
154 #update_locals(self, locals())
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
155
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
156
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
157
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
158 self.layers = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
159 self.pretrain_functions = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
160 self.params = []
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
161 self.n_layers = len(conv_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
162 self.mlp_n_layers = len(mlp_hidden_layers_sizes)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
163
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
164 self.x = T.matrix('x') # the data is presented as rasterized images
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
165 self.y = T.ivector('y') # the labels are presented as 1D vector of
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
166
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
167 for i in xrange( self.n_layers ):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
168 filter_shape=conv_hidden_layers_sizes[i][0]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
169 image_shape=conv_hidden_layers_sizes[i][1]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
170 max_poolsize=conv_hidden_layers_sizes[i][2]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
171
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
172 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
173 layer_input=self.x.reshape((batch_size, 1, 32, 32))
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
174 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
175 layer_input=self.layers[-1].output
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
176
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
177 layer = LeNetConvPoolLayer(rng, input=layer_input,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
178 image_shape=image_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
179 filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
180 poolsize=max_poolsize)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
181 print 'Convolutional layer', str(i+1), 'created'
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
182
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
183 self.layers += [layer]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
184 self.params += layer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
185
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
186 da_layer = dA_conv(corruption_level = corruption_levels[0],
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
187 input = layer_input,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
188 shared_W = layer.W, shared_b = layer.b,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
189 filter_shape=filter_shape,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
190 image_shape = image_shape, num=i , batch_size=batch_size)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
191
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
192 gparams = T.grad(da_layer.cost, da_layer.params)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
193
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
194 updates = {}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
195 for param, gparam in zip(da_layer.params, gparams):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
196 updates[param] = param - gparam * pretrain_lr
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
197
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
198 update_fn = theano.function([self.x], da_layer.cost, updates = updates)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
199
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
200 self.pretrain_functions += [update_fn]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
201
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
202 for i in xrange( self.mlp_n_layers ):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
203 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
204 input_size = n_ins_mlp
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
205 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
206 input_size = mlp_hidden_layers_sizes[i-1]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
207
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
208 if i == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
209 if len( self.layers ) == 0 :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
210 layer_input=self.x
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
211 else :
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
212 layer_input = self.layers[-1].output.flatten(2)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
213 else:
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
214 layer_input = self.layers[-1].output
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
215
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
216 layer = SigmoidalLayer(rng, layer_input, input_size,
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
217 mlp_hidden_layers_sizes[i] )
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
218
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
219 self.layers += [layer]
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
220 self.params += layer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
221
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
222 print 'MLP layer', str(i+1), 'created'
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
223
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
224 self.logLayer = LogisticRegression(input=self.layers[-1].output, \
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
225 n_in=mlp_hidden_layers_sizes[-1], n_out=n_out)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
226
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
227
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
228 self.params += self.logLayer.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
229 self.all_params = self.params
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
230 cost = self.logLayer.negative_log_likelihood(self.y)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
231
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
232 gparams = T.grad(cost, self.params)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
233
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
234 updates = {}
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
235 for param,gparam in zip(self.params, gparams):
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
236 updates[param] = param - gparam*finetune_lr
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
237
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
238 self.finetune = theano.function([self.x, self.y], cost, updates = updates)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
239
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
240 self.errors = self.logLayer.errors(self.y)
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
241
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
242
31641a84e0ae Initial commit for the experimental setup of the denoising convolutional network
humel
parents:
diff changeset
243